diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index a9e5906b..fcd61de4 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -121,12 +121,16 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help - CI/CD validation 阶段只能验证部署对象的 `/health` 端点和必要 provenance;禁止在 CI/CD gate 中运行 web-probe、Playwright、远程浏览器截图、用户路径 E2E 或等价重型业务探针。业务/用户入口验证只能作为发布后的独立 post-deploy validation 证据,不得阻塞 CI/CD 一键交付。 - 产品功能配置 schema 的 CI/CD 合同: - repo 内唯一约定路径固定为 `config/feature-config.schema.json`; + - schema 固定使用 JSON Schema Draft 2020-12,并由 lockfile 锁定的完整结构化 validator 校验;禁止用正则、浅层字段遍历或关闭整体 strict 代替; - schema 只描述产品功能配置,不得承载数据 authority、事件路径、投影实现或交付路径选择; - - 一个产品功能只允许一个规范配置变量,禁止在 schema、renderer、Deployment env 或应用解析器中保留别名、反向变量或重复开关; - - CI/CD 必须检查 schema 语法、配置值匹配和功能到配置变量的一对一关系,并输出有界 typed warning; - - schema 缺失、非法、配置不匹配或发现重复变量时,只允许记录 warning;PipelineRun、artifact、GitOps promote、Argo reconcile、runtime rollout 和 `/health` closeout 必须继续; + - 每个顶层 `properties` 成员必须声明唯一非空 `x-unidesk-feature`;一个产品功能只允许一个规范配置 property,禁止别名、反向变量或重复开关; + - candidate snapshot 必须由正常 source-artifact 阶段从 owning YAML 或已渲染 workload 显式生成;禁止读取 runner `process.env` 作为产品配置 authority; + - HWLAB、AgentRun 与 Sentinel consumer 必须复用同一 shared helper,并在正常 source-artifact/GitOps publish 路径执行,不得只挂到某个 consumer 的专属后处理; + - CI/CD 必须检查 schema 语法、嵌套配置值匹配和功能到配置 property 的一对一关系,并输出有界 typed warning;输入快照缺失使用独立 typed warning,不得与 schema 非法混淆; + - schema 缺失、非法、candidate 缺失、配置不匹配、重复 property、validator/依赖异常或 OTel 导出失败时,只允许记录 warning;embedded 命令必须以 `0` 退出,PipelineRun、artifact、GitOps promote、Argo reconcile、runtime rollout 和 `/health` closeout 必须继续; - schema 校验器只读,禁止写回配置、注入默认值、删除变量、关闭功能、切换架构路径或执行任何补救操作;运行时必须原样接收本次待发布配置; - - schema warning 必须进入 CI/CD OTel span 与 `status|history|debug-step` 的同构有界投影,但不得改变成功/失败终态。 + - schema warning 必须实际导出 CI/CD OTel span/event;endpoint、serviceName、sampling 和 exporter timeout 只读 `config/platform-infra/pipelines-as-code.yaml#observability` 与精确 repository params,零/多 repository 匹配在 renderer 生成期 fail-closed; + - OTel exporter 自身失败继续输出 `blocking=false` warning;warning 同时进入 `status|history|debug-step` 的同构有界投影,但不得改变成功/失败终态。 - 任一 CI/CD 阶段或总耗时超过 owning YAML 预算时,不要继续死等或把超长等待视为正常;先输出阶段耗时分解,并优先从 env reuse、git mirror、BuildKit/cache、GitOps/Argo watch 和 runtime readiness 探测方向优化后再继续交付。 - 仅 legacy lane 的 node-scoped `trigger-current --wait` 可以把 source sync、pre/post flush、PipelineRun、GitOps/Argo、runtime readiness 和 `/health` closeout 放进同一端到端 YAML 预算;超预算时由 CLI 输出阶段分解、Argo target revision、runtime/public 状态和 TaskRun/Pod drill-down。PaC migrated consumer 禁止执行 `trigger-current`,也禁止手动串联状态、sync 或 flush 命令补齐交付。 - node-scoped `control-plane status` 可见性规则: diff --git a/.tekton/web-probe-sentinel-jd01-pac.yaml b/.tekton/web-probe-sentinel-jd01-pac.yaml index 83fafa54..6222e326 100644 --- a/.tekton/web-probe-sentinel-jd01-pac.yaml +++ b/.tekton/web-probe-sentinel-jd01-pac.yaml @@ -68,6 +68,7 @@ spec: bun scripts/native/cicd/render-sentinel-publish-task.ts \ --node JD01 \ --lane v03 \ + --repository-id sentinel-jd01-v03 \ --sentinel jd01-web-probe-sentinel \ --pipeline-run "$(context.pipelineRun.name)" \ --source-commit "{{revision}}" \ diff --git a/.tekton/web-probe-sentinel-nc01-pac.yaml b/.tekton/web-probe-sentinel-nc01-pac.yaml index 6ec53bce..5a8126c7 100644 --- a/.tekton/web-probe-sentinel-nc01-pac.yaml +++ b/.tekton/web-probe-sentinel-nc01-pac.yaml @@ -68,6 +68,7 @@ spec: bun scripts/native/cicd/render-sentinel-publish-task.ts \ --node NC01 \ --lane v03 \ + --repository-id sentinel-nc01-v03 \ --sentinel nc01-web-probe-sentinel \ --pipeline-run "$(context.pipelineRun.name)" \ --source-commit "{{revision}}" \ diff --git a/bun.lock b/bun.lock index 29d6e93c..3b8c817b 100644 --- a/bun.lock +++ b/bun.lock @@ -4,6 +4,10 @@ "workspaces": { "": { "name": "unidesk-root", + "dependencies": { + "ajv-dist": "8.17.1", + "yaml": "2.3.4", + }, "devDependencies": { "@types/bun": "latest", "@types/node": "latest", @@ -98,6 +102,8 @@ "adler-32": ["adler-32@1.3.1", "", {}, "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A=="], + "ajv-dist": ["ajv-dist@8.17.1", "", {}, "sha512-KzJwANMzTTR/RERGnkx+bHzmxIfMTPMMv7+cH1d6Lx9UQ7BZyhiieq4hnO5lRuBWOtYTUL8hyWs7RJYI/45Rtg=="], + "bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="], "cfb": ["cfb@1.2.2", "", { "dependencies": { "adler-32": "~1.3.0", "crc-32": "~1.2.0" } }, "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA=="], @@ -184,6 +190,8 @@ "xlsx": ["xlsx@0.18.5", "", { "dependencies": { "adler-32": "~1.3.0", "cfb": "~1.2.1", "codepage": "~1.15.0", "crc-32": "~1.2.1", "ssf": "~0.11.2", "wmf": "~1.0.1", "word": "~0.3.0" }, "bin": { "xlsx": "bin/xlsx.njs" } }, "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ=="], + "yaml": ["yaml@2.3.4", "", {}, "sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA=="], + "vite/fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], } } diff --git a/config/feature-config.schema.json b/config/feature-config.schema.json new file mode 100644 index 00000000..7b604882 --- /dev/null +++ b/config/feature-config.schema.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://unidesk.pikapython.com/schema/feature-config.schema.json", + "title": "UniDesk Product Feature Configuration", + "type": "object", + "properties": { + "webProbeSentinel": { + "x-unidesk-feature": "web-probe-sentinel", + "type": "object", + "required": [ + "enabled", + "scheduler" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "scheduler": { + "type": "object", + "required": [ + "intervalSeconds" + ], + "properties": { + "intervalSeconds": { + "type": "integer", + "minimum": 30 + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index f5dc6d10..0e718449 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -210,14 +210,6 @@ lanes: codexStdioSupervisor: repo-owned kafkaEventBridge: enabled: true - authority: transactional-projection - features: - directPublish: false - liveKafkaSse: false - kafkaRefreshReplay: false - transactionalProjector: true - projectionOutboxRelay: true - projectionRealtime: true transactionalProjector: heartbeatIntervalMs: 2000 projectionOutboxRelay: @@ -242,7 +234,6 @@ lanes: agentRunEventTopic: agentrun.event.v1 hwlabEventTopic: hwlab.event.v1 clientId: hwlab-v03-cloud-api - directPublishConsumerGroupId: hwlab-v03-agentrun-event-direct-publish transactionalProjectorConsumerGroupId: hwlab-v03-agentrun-event-projector hwlabEventConsumerGroupId: hwlab-v03-workbench-live-sse publicExposure: diff --git a/config/unidesk-host-k8s.yaml b/config/unidesk-host-k8s.yaml index 2d9a28ea..774df30f 100644 --- a/config/unidesk-host-k8s.yaml +++ b/config/unidesk-host-k8s.yaml @@ -23,17 +23,33 @@ delivery: changeDetection: runtimePaths: - .tekton/unidesk-host-pac.yaml + - config/feature-config.schema.json - config/hwlab-node-lanes.yaml - config/platform-infra/pipelines-as-code.yaml - config/unidesk-host-k8s.yaml + - bun.lock + - package.json - scripts/native/cicd/build-unidesk-host-image.sh + - scripts/native/cicd/feature-config-schema-warning.mjs + - scripts/native/cicd/otel-runtime-config.ts + - scripts/native/cicd/pac-status-evaluator.cjs - scripts/native/cicd/prepare-unidesk-host-release.mjs - scripts/native/cicd/publish-unidesk-host-gitops.mjs + - scripts/native/cicd/render-sentinel-publish-task.ts - scripts/native/deploy/render-unidesk-host-service.mjs - scripts/native/hwlab/runtime-gitops-observability.mjs - scripts/native/hwlab/runtime-gitops-postprocess.mjs - scripts/native/hwlab/runtime-gitops-scripts-configmap.mjs - scripts/native/hwlab/runtime-gitops-verify.mjs + - scripts/src/agentrun-manifests.ts + - scripts/src/hwlab-node/render.ts + - scripts/src/hwlab-node/web-probe.ts + - scripts/src/hwlab-node-web-observe-runner-realtime-source.ts + - scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts + - scripts/src/pac-feature-config-schema-stage.ts + - scripts/src/platform-infra-pac-feature-config-projection.ts + - scripts/src/platform-infra-pipelines-as-code-source-artifact.ts + - scripts/src/platform-infra-pipelines-as-code.ts - scripts/src/config.ts - scripts/src/hwlab-node-lanes.ts - scripts/src/yaml-composition.ts diff --git a/package.json b/package.json index f4ffa627..f30f95cb 100644 --- a/package.json +++ b/package.json @@ -19,5 +19,9 @@ "vite": "^8.1.0", "vue": "3", "xlsx": "^0.18.5" + }, + "dependencies": { + "ajv-dist": "8.17.1", + "yaml": "2.3.4" } } diff --git a/scripts/native/cicd/feature-config-schema-warning.mjs b/scripts/native/cicd/feature-config-schema-warning.mjs new file mode 100644 index 00000000..63d537e8 --- /dev/null +++ b/scripts/native/cicd/feature-config-schema-warning.mjs @@ -0,0 +1,382 @@ +#!/usr/bin/env node +import { randomBytes } from "node:crypto"; +import { existsSync, lstatSync, readFileSync, readdirSync } from "node:fs"; +import path from "node:path"; +import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; + +const require = createRequire(import.meta.url); +const cwdRequire = createRequire(path.join(process.cwd(), "package.json")); +const YAML = globalThis.unideskYaml ?? optionalRequire("yaml") ?? optionalRequire("yaml", cwdRequire); + +export const FEATURE_CONFIG_SCHEMA_PATH = "config/feature-config.schema.json"; + +export function validateFeatureConfigSchema(options = {}) { + const repoDir = path.resolve(options.repoDir ?? process.cwd()); + const schemaPath = FEATURE_CONFIG_SCHEMA_PATH; + const absoluteSchemaPath = path.join(repoDir, schemaPath); + if (!existsSync(absoluteSchemaPath)) return result("feature-config-schema-missing", "schema file is missing", 1, null); + + let schema; + try { + schema = JSON.parse(readFileSync(absoluteSchemaPath, "utf8")); + } catch (error) { + return result("feature-config-schema-invalid", `schema JSON parse failed: ${errorMessage(error)}`, 1, null); + } + + const featureErrors = featureDeclarationErrors(schema); + if (featureErrors.invalid.length > 0) return result("feature-config-schema-invalid", featureErrors.invalid[0], featureErrors.invalid.length, null); + if (featureErrors.duplicates.length > 0) return result("feature-config-variable-duplicate", featureErrors.duplicates[0], featureErrors.duplicates.length, null); + + let validate; + try { + const Ajv2020 = ajv2020(); + const ajv = new Ajv2020({ allErrors: true, strict: true, validateFormats: false }); + ajv.addKeyword({ keyword: "x-unidesk-feature", schemaType: "string", valid: true }); + validate = ajv.compile(schema); + } catch (error) { + return result("feature-config-schema-invalid", errorMessage(error), 1, null); + } + + let candidate; + try { + candidate = options.candidate ?? renderedWorkloadCandidate({ + renderedRoot: options.renderedRoot, + propertyNames: Object.keys(schema.properties), + }); + } catch (error) { + return result("feature-config-validator-failure", errorMessage(error), 1, null); + } + if (candidate === null || !isRecord(candidate.values) || Object.keys(candidate.values).length === 0) { + return result("feature-config-input-missing", "rendered workload candidate snapshot is missing", 1, candidate?.summary ?? null); + } + + if (!validate(candidate.values)) { + const errors = Array.isArray(validate.errors) ? validate.errors : []; + return result("feature-config-value-mismatch", boundedAjvError(errors[0]), errors.length || 1, candidate.summary); + } + return result("feature-config-schema-valid", null, 0, candidate.summary); +} + +export function emitFeatureConfigSchemaValidation(options = {}) { + const validation = validateFeatureConfigSchema(options); + process.stderr.write(`${JSON.stringify(validation)}\n`); + return validation; +} + +export async function runFeatureConfigSchemaValidation(options = {}) { + let validation; + try { + validation = emitFeatureConfigSchemaValidation(options); + } catch (error) { + validation = result("feature-config-validator-failure", errorMessage(error), 1, null); + process.stderr.write(`${JSON.stringify(validation)}\n`); + } + try { + const endpoint = options.otelEndpoint ?? process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT ?? ""; + const timeoutMs = positiveInteger(options.otelTimeoutMs ?? process.env.OTEL_EXPORTER_TIMEOUT_MS); + const samplingRatio = samplingRatioValue(options.otelSamplingRatio ?? process.env.OTEL_TRACES_SAMPLER_ARG); + if (endpoint.length === 0) { + const exportWarning = otelWarning("ENDPOINT_MISSING", null, false, null, timeoutMs); + process.stderr.write(`${JSON.stringify(exportWarning)}\n`); + return { validation, otelPayload: null, otelWarning: exportWarning }; + } + if (timeoutMs === null) { + const exportWarning = otelWarning("TIMEOUT_MISSING", null, false, null, null); + process.stderr.write(`${JSON.stringify(exportWarning)}\n`); + return { validation, otelPayload: null, otelWarning: exportWarning }; + } + const otelPayload = featureConfigOtelPayload(validation, { + serviceName: options.otelServiceName ?? process.env.OTEL_SERVICE_NAME ?? "unidesk-cicd", + consumer: options.consumer ?? process.env.UNIDESK_PAC_CONSUMER ?? null, + samplingRatio, + traceparent: options.traceparent ?? process.env.TRACEPARENT ?? process.env.traceparent ?? "", + nowMs: options.nowMs, + traceId: options.traceId, + spanId: options.spanId, + }); + const exportWarning = await exportFeatureConfigOtel(endpoint, otelPayload, timeoutMs, options.fetchImpl ?? fetch); + if (exportWarning !== null) process.stderr.write(`${JSON.stringify(exportWarning)}\n`); + return { validation, otelPayload, otelWarning: exportWarning }; + } catch (error) { + const exportWarning = otelWarning("EXPORT_INTERNAL_FAILURE", null, false, error, positiveInteger(options.otelTimeoutMs ?? process.env.OTEL_EXPORTER_TIMEOUT_MS)); + process.stderr.write(`${JSON.stringify(exportWarning)}\n`); + return { validation, otelPayload: null, otelWarning: exportWarning }; + } +} + +export function featureConfigOtelPayload(validation, options = {}) { + const parent = parseTraceparent(options.traceparent); + const traceId = options.traceId ?? parent.traceId ?? randomBytes(16).toString("hex"); + const spanId = options.spanId ?? randomBytes(8).toString("hex"); + const nowMs = options.nowMs ?? Date.now(); + const startTimeUnixNano = String(BigInt(nowMs) * 1_000_000n); + return { + resourceSpans: [{ + resource: { attributes: otelAttributes({ + "service.name": options.serviceName ?? "unidesk-cicd", + "unidesk.pac.consumer": options.consumer ?? null, + "unidesk.otel.sampling_ratio": options.samplingRatio ?? null, + "unidesk.values_redacted": true, + }) }, + scopeSpans: [{ + scope: { name: "unidesk.cicd.feature-config", version: "v1" }, + spans: [{ + traceId, + spanId, + ...(parent.spanId === null ? {} : { parentSpanId: parent.spanId }), + name: "cicd.feature_config.schema", + kind: 1, + startTimeUnixNano, + endTimeUnixNano: startTimeUnixNano, + attributes: otelAttributes({ + "feature_config.warning": validation.warning, + "feature_config.blocking": false, + "feature_config.code": validation.code, + "feature_config.schema_path": validation.schemaPath, + "feature_config.error_count": validation.errorCount, + }), + events: [{ + timeUnixNano: startTimeUnixNano, + name: "feature-config-schema-validation", + attributes: otelAttributes({ + "feature_config.warning": validation.warning, + "feature_config.code": validation.code, + "feature_config.first_error": validation.firstError, + }), + }], + status: { code: 1 }, + }], + }], + }], + }; +} + +async function exportFeatureConfigOtel(endpoint, payload, timeoutMs, fetchImpl) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetchImpl(endpoint, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(payload), + signal: controller.signal, + }); + if (response.ok) return null; + return otelWarning(`HTTP_${response.status}`, response.status, false, null, timeoutMs); + } catch (error) { + return otelWarning(error instanceof Error && error.name === "AbortError" ? "TIMEOUT" : "EXPORT_FAILED", null, error instanceof Error && error.name === "AbortError", error, timeoutMs); + } finally { + clearTimeout(timeout); + } +} + +function featureDeclarationErrors(schema) { + if (!isRecord(schema) || schema.type !== "object" || !isRecord(schema.properties)) { + return { invalid: ["schema root must be an object schema with properties"], duplicates: [] }; + } + const invalid = []; + const variablesByFeature = new Map(); + for (const [propertyName, propertySchema] of Object.entries(schema.properties)) { + if (!isRecord(propertySchema)) { + invalid.push(`property ${propertyName} schema must be an object`); + continue; + } + const feature = propertySchema["x-unidesk-feature"]; + if (typeof feature !== "string" || feature.trim().length === 0) { + invalid.push(`property ${propertyName} must declare x-unidesk-feature`); + continue; + } + const variables = variablesByFeature.get(feature.trim()) ?? []; + variables.push(propertyName); + variablesByFeature.set(feature.trim(), variables); + } + const duplicates = [...variablesByFeature.entries()] + .filter(([, variables]) => variables.length > 1) + .map(([feature, variables]) => `feature ${feature} maps to multiple properties: ${variables.join(",")}`); + return { invalid, duplicates }; +} + +function renderedWorkloadCandidate({ renderedRoot, propertyNames }) { + if (typeof renderedRoot !== "string" || renderedRoot.length === 0 || !existsSync(renderedRoot) || YAML === null) return null; + const wanted = new Set(propertyNames); + const values = {}; + let fileCount = 0; + let workloadCount = 0; + let matchedValueCount = 0; + for (const file of listStructuredFiles(renderedRoot)) { + fileCount += 1; + for (const document of parseDocuments(file)) { + for (const item of kubernetesItems(document)) { + const podSpec = item?.spec?.template?.spec; + if (!isRecord(podSpec)) continue; + workloadCount += 1; + for (const group of ["containers", "initContainers"]) { + for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) { + for (const env of Array.isArray(container?.env) ? container.env : []) { + if (!wanted.has(env?.name) || typeof env?.value !== "string") continue; + values[env.name] = parseRenderedValue(env.value); + matchedValueCount += 1; + } + } + } + } + } + } + return { + values, + summary: { + source: "rendered-workload-env", + renderedRoot: path.relative(process.cwd(), path.resolve(renderedRoot)) || ".", + fileCount, + workloadCount, + matchedValueCount, + propertyCount: propertyNames.length, + valuesPrinted: false, + }, + }; +} + +function listStructuredFiles(root) { + const maxFiles = 2_000; + const maxDirectories = 2_000; + const files = []; + const pending = [path.resolve(root)]; + let directoryCount = 0; + while (pending.length > 0) { + const current = pending.pop(); + directoryCount += 1; + if (directoryCount > maxDirectories) throw new Error(`rendered workload directory limit exceeded: ${maxDirectories}`); + for (const name of readdirSync(current)) { + const file = path.join(current, name); + const stat = lstatSync(file); + if (stat.isSymbolicLink()) continue; + if (stat.isDirectory()) pending.push(file); + else if (/\.(?:ya?ml|json)$/iu.test(name)) { + files.push(file); + if (files.length > maxFiles) throw new Error(`rendered workload file limit exceeded: ${maxFiles}`); + } + } + } + return files.sort(); +} + +function parseDocuments(file) { + const text = readFileSync(file, "utf8"); + if (/\.json$/iu.test(file)) { + try { return [JSON.parse(text)]; } catch { return []; } + } + try { return YAML.parseAllDocuments(text).map((document) => document.toJS()).filter((document) => document !== null); } catch { return []; } +} + +function kubernetesItems(document) { + return document?.kind === "List" && Array.isArray(document.items) ? document.items : [document]; +} + +function parseRenderedValue(value) { + try { return JSON.parse(value); } catch { return value; } +} + +function boundedAjvError(error) { + if (!isRecord(error)) return "candidate does not match schema"; + const instancePath = typeof error.instancePath === "string" ? error.instancePath : ""; + const keyword = typeof error.keyword === "string" ? error.keyword : "validation"; + const message = typeof error.message === "string" ? error.message : "failed"; + return `${instancePath || "/"} ${keyword} ${message}`; +} + +function result(code, firstError, errorCount, candidate) { + const warning = errorCount > 0; + const boundedError = typeof firstError === "string" ? firstError.replace(/\s+/gu, " ").trim().slice(0, 240) : null; + return { + event: "feature-config-schema-validation", + warning, + blocking: false, + code, + schemaPath: FEATURE_CONFIG_SCHEMA_PATH, + errorCount, + firstError: boundedError, + candidate, + mutation: false, + valuesPrinted: false, + otel: { + spanEvent: "cicd.feature_config.schema", + attributes: { + "feature_config.warning": warning, + "feature_config.code": code, + "feature_config.schema_path": FEATURE_CONFIG_SCHEMA_PATH, + "feature_config.error_count": errorCount, + "feature_config.blocking": false, + }, + }, + }; +} + +function optionalRequire(name, loader = require) { + try { return loader(name); } catch { return null; } +} + +function ajv2020() { + return globalThis.unideskAjv2020 ?? bundledAjv(process.env.UNIDESK_AJV2020_BUNDLE) ?? require("ajv-dist/dist/ajv2020.bundle.js"); +} + +function bundledAjv(file) { + if (typeof file !== "string" || file.length === 0 || !existsSync(file)) return null; + const module = { exports: {} }; + Function("module", "exports", readFileSync(file, "utf8"))(module, module.exports); + return module.exports.default ?? module.exports; +} + +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function errorMessage(error) { + return error instanceof Error ? error.message : String(error); +} + +function positiveInteger(value) { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : null; +} + +function samplingRatioValue(value) { + const parsed = Number(value); + return Number.isFinite(parsed) && parsed >= 0 && parsed <= 1 ? parsed : null; +} + +function parseTraceparent(value) { + const match = /^00-([0-9a-f]{32})-([0-9a-f]{16})-[0-9a-f]{2}$/iu.exec(String(value)); + return { traceId: match?.[1]?.toLowerCase() ?? null, spanId: match?.[2]?.toLowerCase() ?? null }; +} + +function otelAttributes(values) { + return Object.entries(values).filter(([, value]) => value !== null && value !== undefined).map(([key, value]) => ({ + key, + value: typeof value === "boolean" + ? { boolValue: value } + : typeof value === "number" + ? { intValue: String(value) } + : { stringValue: String(value) }, + })); +} + +function otelWarning(causeCode, httpStatus, timedOut, error, timeoutMs) { + return { + event: "cicd.otel.export", + warning: true, + blocking: false, + causeCode, + ...(httpStatus === null ? {} : { httpStatus }), + timedOut, + errorType: error instanceof Error ? error.name : null, + timeoutMs, + valuesRedacted: true, + }; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + const renderedRootIndex = process.argv.indexOf("--rendered-root"); + await runFeatureConfigSchemaValidation({ renderedRoot: renderedRootIndex === -1 ? null : process.argv[renderedRootIndex + 1] }); + process.exitCode = 0; +} diff --git a/scripts/native/cicd/feature-config-schema-warning.test.ts b/scripts/native/cicd/feature-config-schema-warning.test.ts new file mode 100644 index 00000000..13f24c66 --- /dev/null +++ b/scripts/native/cicd/feature-config-schema-warning.test.ts @@ -0,0 +1,211 @@ +import { afterEach, expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + featureConfigOtelPayload, + runFeatureConfigSchemaValidation, + validateFeatureConfigSchema, +} from "./feature-config-schema-warning.mjs"; + +const roots: string[] = []; + +afterEach(() => { + while (roots.length > 0) rmSync(roots.pop()!, { recursive: true, force: true }); +}); + +test("valid nested Draft 2020-12 candidate remains warning-free", () => { + const root = fixture(schema({ + tracePanel: feature("trace-panel", { + type: "object", + required: ["enabled", "views"], + properties: { + enabled: { type: "boolean" }, + views: { type: "array", minItems: 1, items: { type: "string", pattern: "^[a-z-]+$" } }, + }, + additionalProperties: false, + }), + }), { tracePanel: '{"enabled":true,"views":["timeline"]}' }); + expect(validateFeatureConfigSchema({ repoDir: root, renderedRoot: join(root, "rendered") })).toMatchObject({ + warning: false, + blocking: false, + code: "feature-config-schema-valid", + errorCount: 0, + mutation: false, + candidate: { source: "rendered-workload-env", matchedValueCount: 1 }, + }); +}); + +test("missing schema is a typed non-blocking warning", () => { + const root = fixture(); + expect(validateFeatureConfigSchema({ repoDir: root, renderedRoot: join(root, "rendered") })).toMatchObject({ + warning: true, + blocking: false, + code: "feature-config-schema-missing", + }); +}); + +test("invalid JSON and unsupported Draft 2020-12 schema are typed warnings", () => { + const invalidJsonRoot = fixture("{"); + expect(validateFeatureConfigSchema({ repoDir: invalidJsonRoot, renderedRoot: join(invalidJsonRoot, "rendered") }).code).toBe("feature-config-schema-invalid"); + + const invalidSchemaRoot = fixture(JSON.stringify({ $schema: "https://json-schema.org/draft/2020-12/schema", type: "object", properties: { tracePanel: { type: "not-a-json-schema-type", "x-unidesk-feature": "trace-panel" } } })); + expect(validateFeatureConfigSchema({ repoDir: invalidSchemaRoot, renderedRoot: join(invalidSchemaRoot, "rendered") }).code).toBe("feature-config-schema-invalid"); +}); + +test("missing rendered candidate is distinct from schema failure", () => { + const root = fixture(schema({ tracePanel: feature("trace-panel", { type: "boolean" }) })); + expect(validateFeatureConfigSchema({ repoDir: root, renderedRoot: join(root, "missing") })).toMatchObject({ + warning: true, + blocking: false, + code: "feature-config-input-missing", + }); +}); + +test("nested value mismatch is bounded and does not disclose the value", () => { + const root = fixture(schema({ tracePanel: feature("trace-panel", { type: "object", required: ["enabled"], properties: { enabled: { const: true } } }) }), { + tracePanel: '{"enabled":false,"secret":"do-not-print"}', + }); + const result = validateFeatureConfigSchema({ repoDir: root, renderedRoot: join(root, "rendered") }); + expect(result).toMatchObject({ warning: true, blocking: false, code: "feature-config-value-mismatch", valuesPrinted: false }); + expect(result.firstError).not.toContain("do-not-print"); +}); + +test("duplicate feature properties are reported without blocking", () => { + const root = fixture(schema({ + tracePanel: feature("trace-panel", { type: "boolean" }), + tracePanelAlias: feature("trace-panel", { type: "boolean" }), + })); + expect(validateFeatureConfigSchema({ repoDir: root, renderedRoot: join(root, "rendered") })).toMatchObject({ + warning: true, + blocking: false, + code: "feature-config-variable-duplicate", + errorCount: 1, + }); +}); + +test("actual OTLP span and event carry redacted validation attributes", () => { + const root = fixture(); + const validation = validateFeatureConfigSchema({ repoDir: root, renderedRoot: join(root, "rendered") }); + const payload = featureConfigOtelPayload(validation, { + serviceName: "unidesk-cicd", + consumer: "hwlab-nc01-v03", + samplingRatio: 1, + nowMs: 1, + traceId: "1".repeat(32), + spanId: "2".repeat(16), + }); + const span = payload.resourceSpans[0].scopeSpans[0].spans[0]; + expect(span.name).toBe("cicd.feature_config.schema"); + expect(span.events[0].name).toBe("feature-config-schema-validation"); + expect(JSON.stringify(payload)).not.toContain("do-not-print"); + expect(JSON.stringify(payload)).toContain("unidesk.otel.sampling_ratio"); +}); + +test("missing endpoint remains a non-blocking OTel warning", async () => { + const root = fixture(); + const result = await runFeatureConfigSchemaValidation({ + repoDir: root, + renderedRoot: join(root, "rendered"), + otelEndpoint: "", + otelTimeoutMs: 300, + }); + expect(result.otelWarning).toMatchObject({ + event: "cicd.otel.export", + warning: true, + blocking: false, + causeCode: "ENDPOINT_MISSING", + timeoutMs: 300, + }); +}); + +test("export failure uses configured timeout and remains non-blocking", async () => { + const root = fixture(); + let requestSignal: AbortSignal | null = null; + const result = await runFeatureConfigSchemaValidation({ + repoDir: root, + renderedRoot: join(root, "rendered"), + otelEndpoint: "http://otel.example.test/v1/traces", + otelServiceName: "unidesk-cicd", + otelSamplingRatio: 1, + otelTimeoutMs: 300, + fetchImpl: async (_input: URL | RequestInfo, init?: RequestInit) => { + requestSignal = init?.signal ?? null; + throw new Error("collector unavailable"); + }, + }); + expect(requestSignal).not.toBeNull(); + expect(result.otelWarning).toMatchObject({ + event: "cicd.otel.export", + warning: true, + blocking: false, + causeCode: "EXPORT_FAILED", + timeoutMs: 300, + }); +}); + +test("unexpected validator failure is typed and never rejects", async () => { + const options: Record = { + repoDir: fixture(schema({ webProbeSentinel: feature("web-probe-sentinel", { type: "object" }) })), + renderedRoot: "unused", + otelEndpoint: "", + otelTimeoutMs: 300, + }; + Object.defineProperty(options, "candidate", { get: () => { throw new Error("unexpected candidate failure"); } }); + const result = await runFeatureConfigSchemaValidation(options); + expect(result.validation).toMatchObject({ + warning: true, + blocking: false, + code: "feature-config-validator-failure", + mutation: false, + }); + expect(result.otelWarning).toMatchObject({ warning: true, blocking: false, causeCode: "ENDPOINT_MISSING" }); +}); + +test("candidate scan ignores directory symlinks and bounds directory traversal", () => { + const root = fixture(schema({ webProbeSentinel: feature("web-probe-sentinel", { type: "boolean" }) }), { + webProbeSentinel: "true", + }); + symlinkSync(join(root, "rendered"), join(root, "rendered", "loop"), "dir"); + expect(validateFeatureConfigSchema({ repoDir: root, renderedRoot: join(root, "rendered") })).toMatchObject({ + warning: false, + code: "feature-config-schema-valid", + }); + for (let index = 0; index <= 2_000; index += 1) mkdirSync(join(root, "rendered", `directory-${index}`)); + expect(validateFeatureConfigSchema({ repoDir: root, renderedRoot: join(root, "rendered") })).toMatchObject({ + warning: true, + blocking: false, + code: "feature-config-validator-failure", + firstError: "rendered workload directory limit exceeded: 2000", + }); +}); + +function fixture(content?: string, env: Record = {}) { + const root = mkdtempSync(join(tmpdir(), "feature-config-schema-")); + roots.push(root); + mkdirSync(join(root, "rendered")); + if (content !== undefined) { + mkdirSync(join(root, "config")); + writeFileSync(join(root, "config", "feature-config.schema.json"), content); + } + writeFileSync(join(root, "rendered", "deployment.yaml"), deployment(env)); + return root; +} + +function schema(properties: Record) { + return JSON.stringify({ + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties, + additionalProperties: false, + }); +} + +function feature(name: string, value: Record) { + return { ...value, "x-unidesk-feature": name }; +} + +function deployment(env: Record) { + const rows = Object.entries(env).map(([name, value]) => ` - name: ${name}\n value: ${JSON.stringify(value)}`).join("\n"); + return `apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: fixture\nspec:\n template:\n spec:\n containers:\n - name: fixture\n env:\n${rows || " []"}\n`; +} diff --git a/scripts/native/cicd/otel-runtime-config.ts b/scripts/native/cicd/otel-runtime-config.ts index 1b18c542..12a0791f 100644 --- a/scripts/native/cicd/otel-runtime-config.ts +++ b/scripts/native/cicd/otel-runtime-config.ts @@ -28,8 +28,7 @@ export interface CicdOtelExportWarning { } export function loadCicdOtelRuntimeConfig( - node: string, - lane: string, + repositoryId: string, env: Record = process.env, ): CicdOtelRuntimeConfig { const root = materializeYamlComposition( @@ -37,7 +36,7 @@ export function loadCicdOtelRuntimeConfig( { label: "platform-infra-pipelines-as-code", stripTemplateKeys: true }, ).value; const observability = record(root.observability, `${pacConfigPath}#observability`); - const repository = repositoryFor(root.repositories, node, lane); + const repository = selectCicdOtelRepository(root.repositories, repositoryId); const params = record(repository.params, `${pacConfigPath}#repositories.${repository.id}.params`); const fallbackCodes: string[] = []; const endpoint = authoritativeValue( @@ -117,12 +116,12 @@ export async function exportCicdOtelPayload( } } -function repositoryFor(value: unknown, node: string, lane: string): Record { +export function selectCicdOtelRepository(value: unknown, repositoryId: string): Record { if (!Array.isArray(value)) throw new Error(`${pacConfigPath}#repositories must be an array`); - const expectedId = `sentinel-${node.toLowerCase()}-${lane.toLowerCase()}`; - const repository = value.find((item) => recordOrNull(item)?.id === expectedId); - if (repository === undefined) throw new Error(`${pacConfigPath} has no repository ${expectedId}`); - return record(repository, `${pacConfigPath}#repositories.${expectedId}`); + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/u.test(repositoryId)) throw new Error(`invalid PaC repository id ${repositoryId}`); + const repositories = value.filter((item) => recordOrNull(item)?.id === repositoryId); + if (repositories.length !== 1) throw new Error(`${pacConfigPath} requires exactly one repository ${repositoryId}; matched ${repositories.length}`); + return record(repositories[0], `${pacConfigPath}#repositories.${repositoryId}`); } function authoritativeValue( diff --git a/scripts/native/cicd/pac-status-evaluator.cjs b/scripts/native/cicd/pac-status-evaluator.cjs index eb79995d..f15cf7a7 100644 --- a/scripts/native/cicd/pac-status-evaluator.cjs +++ b/scripts/native/cicd/pac-status-evaluator.cjs @@ -31,6 +31,35 @@ function exactRuntimeGitopsVerifyMarker(records) { && item.reason === "no-build-no-rollout-plan-gitops-verify") || null; } +function featureConfigSchemaObservation(records) { + const item = [...records].reverse().find((recordValue) => recordValue.event === "feature-config-schema-validation") || null; + if (item === null) return null; + const candidate = record(item.candidate); + const code = stringOrNull(item.code); + const schemaPath = stringOrNull(item.schemaPath); + const firstError = stringOrNull(item.firstError); + return { + warning: item.warning === true, + blocking: false, + code: code === null ? "feature-config-schema-invalid" : code.slice(0, 100), + schemaPath: schemaPath === null ? "config/feature-config.schema.json" : schemaPath.slice(0, 200), + errorCount: Number.isInteger(item.errorCount) ? Math.max(0, item.errorCount) : null, + firstError: firstError === null ? null : firstError.replace(/\s+/gu, " ").trim().slice(0, 240), + candidate: Object.keys(candidate).length === 0 ? null : { + source: stringOrNull(candidate.source), + renderedRoot: stringOrNull(candidate.renderedRoot), + fileCount: Number.isInteger(candidate.fileCount) ? candidate.fileCount : null, + workloadCount: Number.isInteger(candidate.workloadCount) ? candidate.workloadCount : null, + matchedValueCount: Number.isInteger(candidate.matchedValueCount) ? candidate.matchedValueCount : null, + propertyCount: Number.isInteger(candidate.propertyCount) ? candidate.propertyCount : null, + valuesPrinted: false, + }, + spanEvent: "cicd.feature_config.schema", + mutation: false, + valuesPrinted: false, + }; +} + const pacContractTasks = new Set(["plan-artifacts", "collect-artifacts", "gitops-promote"]); function firstString(...values) { @@ -368,6 +397,7 @@ function terminalSource(task, marker, markerStatus = "skipped", markerSource = " function extractPacSourceObservation(recordsInput) { const records = Array.isArray(recordsInput) ? recordsInput.map(record) : []; + const featureConfigSchema = featureConfigSchemaObservation(records); const plan = [...records].reverse().find((item) => item.event === "pac-delivery-plan" || item.event === "g14-ci-plan") || null; const collectMarker = exactSkipMarker(records, "collect-artifacts"); const promoteMarker = exactSkipMarker(records, "gitops-promote"); @@ -461,6 +491,7 @@ function extractPacSourceObservation(recordsInput) { ? terminalSource(promoteTask, promoteMarker) : terminalSource(promoteTask, runtimeGitopsVerifyMarker, "continuing", "runtime-gitops-verify-structured-log"), }, + featureConfigSchema, valuesPrinted: false, }; } diff --git a/scripts/native/cicd/publish-unidesk-host-gitops.mjs b/scripts/native/cicd/publish-unidesk-host-gitops.mjs index acb100b7..9e9e312a 100755 --- a/scripts/native/cicd/publish-unidesk-host-gitops.mjs +++ b/scripts/native/cicd/publish-unidesk-host-gitops.mjs @@ -75,6 +75,8 @@ export function renderGitOpsResources(gitops, sourceRoot) { const manifest = Bun.YAML.stringify(runtimeGitopsScriptsConfigMap({ overlay, scriptsDir: resolve(sourceRoot, "scripts/native/hwlab"), + featureConfigValidatorPath: resolve(sourceRoot, "scripts/native/cicd/feature-config-schema-warning.mjs"), + ajvBundlePath: resolve(sourceRoot, "node_modules/ajv-dist/dist/ajv2020.min.js"), namespace: required(resource.namespace, `${pathLabel}.namespace`), configMapName, })); diff --git a/scripts/native/cicd/render-sentinel-publish-task.test.ts b/scripts/native/cicd/render-sentinel-publish-task.test.ts index cd562c4c..aa55534d 100644 --- a/scripts/native/cicd/render-sentinel-publish-task.test.ts +++ b/scripts/native/cicd/render-sentinel-publish-task.test.ts @@ -3,7 +3,7 @@ import { mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { rootPath } from "../../src/config"; -import { exportCicdOtelPayload, loadCicdOtelRuntimeConfig } from "./otel-runtime-config"; +import { exportCicdOtelPayload, loadCicdOtelRuntimeConfig, selectCicdOtelRepository } from "./otel-runtime-config"; const roots: string[] = []; @@ -12,7 +12,7 @@ afterEach(() => { }); test("unexpanded Repository inputs fall back to materialized owning YAML", () => { - const config = loadCicdOtelRuntimeConfig("NC01", "v03", { + const config = loadCicdOtelRuntimeConfig("sentinel-nc01-v03", { OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "{{otel_traces_endpoint}}", OTEL_SERVICE_NAME: "{{otel_service_name}}", OTEL_TRACES_SAMPLER_ARG: "{{otel_sampling_ratio}}", @@ -34,7 +34,7 @@ test("unexpanded Repository inputs fall back to materialized owning YAML", () => }); test("valid but mismatched runtime inputs cannot override owning YAML", () => { - const config = loadCicdOtelRuntimeConfig("NC01", "v03", { + const config = loadCicdOtelRuntimeConfig("sentinel-nc01-v03", { OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://collector.example.invalid:4318/v1/traces", OTEL_SERVICE_NAME: "different-service", OTEL_TRACES_SAMPLER_ARG: "0.5", @@ -53,7 +53,7 @@ test("valid but mismatched runtime inputs cannot override owning YAML", () => { }); test("legal owning endpoint and non-blocking exporter failure share the bounded helper", async () => { - const config = loadCicdOtelRuntimeConfig("NC01", "v03", { + const config = loadCicdOtelRuntimeConfig("sentinel-nc01-v03", { OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces", OTEL_SERVICE_NAME: "unidesk-cicd-sentinel-nc01", OTEL_TRACES_SAMPLER_ARG: "1", @@ -81,6 +81,14 @@ test("legal owning endpoint and non-blocking exporter failure share the bounded }); }); +test("repository identity selection fails closed on zero or multiple exact matches", () => { + expect(() => selectCicdOtelRepository([], "sentinel-nc01-v03")).toThrow("matched 0"); + expect(() => selectCicdOtelRepository([ + { id: "sentinel-nc01-v03", params: {} }, + { id: "sentinel-nc01-v03", params: {} }, + ], "sentinel-nc01-v03")).toThrow("matched 2"); +}); + test("actual renderer replaces unexpanded inputs without changing business exit", async () => { const result = await render({ OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "{{otel_traces_endpoint}}" }, false); expect(result.exitCode).toBe(0); @@ -89,6 +97,12 @@ test("actual renderer replaces unexpanded inputs without changing business exit" expect(result.stderr).toContain('"endpointHostname":"otel-collector.platform-infra.svc.cluster.local"'); expect(result.stderr).not.toContain("{{otel_traces_endpoint}}"); expect(readFileSync(join(result.outputDir, "source.sh"), "utf8")).toContain("http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces"); + const publish = readFileSync(join(result.outputDir, "publish.sh"), "utf8"); + expect(publish).toContain("UNIDESK_PAC_CONSUMER='sentinel-nc01-v03'"); + expect(publish).toContain("feature-config-schema-validation"); + expect(publish).toContain("cicd.feature_config.schema"); + expect(publish).toContain("OTEL_EXPORTER_TIMEOUT_MS='300'"); + expect(publish).toContain("feature-config-validator-process-failure"); }); async function render(env: Record, traced = true) { @@ -100,6 +114,7 @@ async function render(env: Record, traced = true) { "scripts/native/cicd/render-sentinel-publish-task.ts", "--node", "NC01", "--lane", "v03", + "--repository-id", "sentinel-nc01-v03", "--sentinel", "nc01-web-probe-sentinel", "--pipeline-run", "hwlab-web-probe-sentinel-nc01-test", "--source-commit", "a".repeat(40), diff --git a/scripts/native/cicd/render-sentinel-publish-task.ts b/scripts/native/cicd/render-sentinel-publish-task.ts index 284f6bb0..87f4c8c8 100644 --- a/scripts/native/cicd/render-sentinel-publish-task.ts +++ b/scripts/native/cicd/render-sentinel-publish-task.ts @@ -9,6 +9,7 @@ import { sentinelPublishSourceShell, } from "../../src/hwlab-node-web-sentinel-cicd-jobs"; import { exportCicdOtelPayload, loadCicdOtelRuntimeConfig } from "./otel-runtime-config"; +import { renderPacFeatureConfigSchemaStage } from "../../src/pac-feature-config-schema-stage"; const options = parseOptions(process.argv.slice(2)); const sourceAuthority = options.sourceAuthority === "gitea-snapshot" ? "gitea-snapshot" : fail("--source-authority must be gitea-snapshot"); @@ -26,7 +27,7 @@ const state = loadSentinelCicdState( ); const outputDir = resolve(options.outputDir); mkdirSync(outputDir, { recursive: true }); -const otelConfig = loadCicdOtelRuntimeConfig(options.node, options.lane); +const otelConfig = loadCicdOtelRuntimeConfig(options.repositoryId); const otelSetup = [ `export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=${shellQuote(otelConfig.endpoint)}`, `export OTEL_EXPORTER_TIMEOUT_MS=${shellQuote(String(otelConfig.timeoutMs))}`, @@ -38,6 +39,12 @@ const traceSetup = [ "if [ -f /workspace/meta/tracestate ]; then TRACESTATE=$(cat /workspace/meta/tracestate); export TRACESTATE; fi", ].join("\n"); const proxySetup = sentinelImageBuildProxyEnv(state).map(({ name, value }) => `export ${name}=${shellQuote(value)}`).join("\n"); +const featureConfigSchemaStage = renderPacFeatureConfigSchemaStage({ + repositoryId: options.repositoryId, + consumer: options.repositoryId, + repoDirExpression: "'/workspace/source'", + renderedRootExpression: '"$gitops_worktree"', +}); writeExecutable("source.sh", [ "#!/bin/sh", "set -eu", @@ -49,7 +56,7 @@ writeExecutable("source.sh", [ "if [ -n \"$incoming_tracestate\" ]; then printf '%s' \"$incoming_tracestate\" > /workspace/meta/tracestate; fi", ].join("\n")); writeExecutable("image-build.sh", ["#!/bin/sh", "set -eu", otelSetup, traceSetup, proxySetup, sentinelPublishImageBuildShell(state, options.pipelineRun)].join("\n")); -writeExecutable("publish.sh", ["#!/bin/sh", "set -eu", otelSetup, traceSetup, proxySetup, sentinelPublishShell(state, options.pipelineRun, true)].join("\n")); +writeExecutable("publish.sh", ["#!/bin/sh", "set -eu", otelSetup, traceSetup, proxySetup, sentinelPublishShell(state, options.pipelineRun, true, featureConfigSchemaStage)].join("\n")); emitConfigFallbackWarning(); await emitOuterSpan(); process.stdout.write(`${JSON.stringify({ ok: true, action: "render-sentinel-publish-task", node: options.node, lane: options.lane, sentinel: options.sentinel, pipelineRun: options.pipelineRun, sourceCommit: options.sourceCommit, outputDir, traceId: traceId(process.env.TRACEPARENT), valuesRedacted: true })}\n`); @@ -60,7 +67,7 @@ function writeExecutable(name: string, content: string): void { chmodSync(path, 0o755); } -function parseOptions(args: string[]): Record<"node" | "lane" | "sentinel" | "pipelineRun" | "sourceCommit" | "sourceStageRef" | "sourceAuthority" | "outputDir", string> { +function parseOptions(args: string[]): Record<"node" | "lane" | "repositoryId" | "sentinel" | "pipelineRun" | "sourceCommit" | "sourceStageRef" | "sourceAuthority" | "outputDir", string> { const values = new Map(); for (let index = 0; index < args.length; index += 2) { const key = args[index]; @@ -72,6 +79,7 @@ function parseOptions(args: string[]): Record<"node" | "lane" | "sentinel" | "pi const result = { node: required("node"), lane: required("lane"), + repositoryId: required("repository-id"), sentinel: required("sentinel"), pipelineRun: required("pipeline-run"), sourceCommit: required("source-commit"), @@ -80,7 +88,7 @@ function parseOptions(args: string[]): Record<"node" | "lane" | "sentinel" | "pi outputDir: required("output-dir"), }; if (!/^[0-9a-f]{40}$/u.test(result.sourceCommit)) fail("--source-commit must be a 40-character lowercase Git commit"); - for (const key of ["node", "lane", "sentinel", "pipelineRun"] as const) if (!/^[A-Za-z0-9._-]+$/u.test(result[key])) fail(`--${key} must be a simple id`); + for (const key of ["node", "lane", "repositoryId", "sentinel", "pipelineRun"] as const) if (!/^[A-Za-z0-9._-]+$/u.test(result[key])) fail(`--${key.replace(/[A-Z]/gu, (letter) => `-${letter.toLowerCase()}`)} must be a simple id`); if (!result.sourceStageRef.startsWith("refs/unidesk/snapshots/")) fail("--source-stage-ref must be an immutable UniDesk snapshot ref"); return result; } diff --git a/scripts/native/hwlab/runtime-gitops-postprocess.mjs b/scripts/native/hwlab/runtime-gitops-postprocess.mjs index a8623f74..604035d7 100644 --- a/scripts/native/hwlab/runtime-gitops-postprocess.mjs +++ b/scripts/native/hwlab/runtime-gitops-postprocess.mjs @@ -62,11 +62,6 @@ function patchCodeAgentRuntimeWorkloads() { if (workloadName === "hwlab-cloud-api" && container.name === "hwlab-cloud-api") { fileChanged = patchCloudApiKafkaEnv(container, kafka) || fileChanged; } - if (workloadName === "hwlab-cloud-web" && container.name === "hwlab-cloud-web") { - fileChanged = setEnvValue(container, "HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED", String(kafka.enabled && kafka.features.liveKafkaSse)) || fileChanged; - fileChanged = setEnvValue(container, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED", String(kafka.enabled && kafka.features.kafkaRefreshReplay)) || fileChanged; - fileChanged = setEnvValue(container, "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED", String(kafka.enabled && kafka.features.projectionRealtime)) || fileChanged; - } } } } @@ -81,27 +76,20 @@ function patchCodeAgentRuntimeWorkloads() { function patchCloudApiKafkaEnv(container, kafka) { let changed = false; changed = setEnvValue(container, "HWLAB_KAFKA_ENABLED", String(kafka.enabled)) || changed; - changed = setEnvValue(container, "HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED", String(kafka.enabled && (kafka.features.directPublish || kafka.features.transactionalProjector))) || changed; - changed = setEnvValue(container, "HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED", String(kafka.enabled && kafka.features.directPublish)) || changed; - changed = setEnvValue(container, "HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED", String(kafka.enabled && kafka.features.liveKafkaSse)) || changed; - changed = setEnvValue(container, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED", String(kafka.enabled && kafka.features.kafkaRefreshReplay)) || changed; - changed = setEnvValue(container, "HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED", String(kafka.enabled && kafka.features.transactionalProjector)) || changed; - changed = setEnvValue(container, "HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED", String(kafka.enabled && kafka.features.projectionOutboxRelay)) || changed; - changed = setEnvValue(container, "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED", String(kafka.enabled && kafka.features.projectionRealtime)) || changed; - changed = setEnvValue(container, "HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID", String(kafka.directPublishConsumerGroupId)) || changed; + changed = setEnvValue(container, "HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED", String(kafka.enabled)) || changed; changed = setEnvValue(container, "HWLAB_KAFKA_PROJECTOR_GROUP_ID", String(kafka.transactionalProjectorConsumerGroupId)) || changed; changed = setEnvValue(container, "HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID", String(kafka.hwlabEventConsumerGroupId)) || changed; - changed = patchOptionalEnvGroup(container, kafka.enabled && kafka.features.transactionalProjector, { + changed = patchOptionalEnvGroup(container, kafka.enabled, { HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS: kafka.transactionalProjector?.heartbeatIntervalMs, }) || changed; - changed = patchOptionalEnvGroup(container, kafka.enabled && kafka.features.projectionOutboxRelay, { + changed = patchOptionalEnvGroup(container, kafka.enabled, { HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS: kafka.projectionOutboxRelay?.intervalMs, HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE: kafka.projectionOutboxRelay?.batchSize, HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS: kafka.projectionOutboxRelay?.leaseMs, HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS: kafka.projectionOutboxRelay?.sendTimeoutMs, HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS: kafka.projectionOutboxRelay?.retryBackoffMs, }) || changed; - changed = patchOptionalEnvGroup(container, kafka.enabled && kafka.features.projectionRealtime, { + changed = patchOptionalEnvGroup(container, kafka.enabled, { HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: kafka.projectionRealtime?.outboxTailBatchSize, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: kafka.projectionRealtime?.sseHeartbeatMs, HWLAB_WORKBENCH_SSE_DRAIN_TIMEOUT_MS: kafka.projectionRealtime?.sseDrainTimeoutMs, diff --git a/scripts/native/hwlab/runtime-gitops-postprocess.test.ts b/scripts/native/hwlab/runtime-gitops-postprocess.test.ts index f0958b6a..d1c36199 100644 --- a/scripts/native/hwlab/runtime-gitops-postprocess.test.ts +++ b/scripts/native/hwlab/runtime-gitops-postprocess.test.ts @@ -11,7 +11,7 @@ afterEach(() => { while (roots.length > 0) rmSync(roots.pop()!, { recursive: true, force: true }); }); -test("patches YAML workloads across ordinary, multi-document, and Kubernetes List files", () => { +test("patches fixed Kafka transport settings without architecture capability env", () => { const root = mkdtempSync(join(tmpdir(), "runtime-gitops-postprocess-")); roots.push(root); const runtimeDir = join(root, "runtime"); @@ -32,23 +32,25 @@ test("patches YAML workloads across ordinary, multi-document, and Kubernetes Lis expect(result.stderr).toContain('"codeAgentRuntimeChanged":true'); const api = Bun.YAML.parse(readFileSync(join(runtimeDir, "cloud-api.yaml"), "utf8")) as any; - expect(authorityValues(api)).toEqual(["false", "false", "false", "true", "true", "true"]); + expect(envValues(api)).toMatchObject({ + HWLAB_KAFKA_ENABLED: "true", + HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED: "true", + HWLAB_KAFKA_PROJECTOR_GROUP_ID: "hwlab-projector", + HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID: "hwlab-events", + }); + expect(architectureEnvNames(api)).toEqual([]); const multiDocs = readFileSync(join(runtimeDir, "cloud-web.yaml"), "utf8").split(/^---$/mu).map((document) => Bun.YAML.parse(document) as any); expect(multiDocs[0].kind).toBe("ConfigMap"); - expect(envValues(multiDocs[1])).toMatchObject({ - HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", - HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", - HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true", - }); + expect(envValues(multiDocs[1])).toEqual({ EXISTING: "preserved" }); const list = Bun.YAML.parse(readFileSync(join(runtimeDir, "list.yaml"), "utf8")) as any; expect(list.kind).toBe("List"); - expect(authorityValues(list.items[0])).toEqual(["false", "false", "false", "true", "true", "true"]); + expect(architectureEnvNames(list.items[0])).toEqual([]); const jsonText = readFileSync(join(runtimeDir, "json-workload.yaml"), "utf8"); expect(jsonText.startsWith("{\n")).toBe(true); - expect(authorityValues(JSON.parse(jsonText))).toEqual(["false", "false", "false", "true", "true", "true"]); + expect(architectureEnvNames(JSON.parse(jsonText))).toEqual([]); const afterFirstRun = snapshot(runtimeDir); const second = spawnSync(process.execPath, [script], { @@ -68,16 +70,11 @@ function overlay(runtimePath: string) { enabled: true, kafkaEventBridge: { enabled: true, - directPublishConsumerGroupId: "hwlab-direct", transactionalProjectorConsumerGroupId: "hwlab-projector", - features: { - directPublish: false, - liveKafkaSse: false, - kafkaRefreshReplay: false, - transactionalProjector: true, - projectionOutboxRelay: true, - projectionRealtime: true, - }, + hwlabEventConsumerGroupId: "hwlab-events", + transactionalProjector: { heartbeatIntervalMs: 2000 }, + projectionOutboxRelay: { intervalMs: 250, batchSize: 100, leaseMs: 30000, sendTimeoutMs: 10000, retryBackoffMs: 1000 }, + projectionRealtime: { outboxTailBatchSize: 100, sseHeartbeatMs: 1000, sseDrainTimeoutMs: 5000 }, }, }, }; @@ -96,16 +93,16 @@ function envValues(workload: any) { return Object.fromEntries(workload.spec.template.spec.containers[0].env.map((item: any) => [item.name, item.value])); } -function authorityValues(workload: any) { - const env = envValues(workload); - return [ - env.HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED, - env.HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED, - env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED, - env.HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED, - env.HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED, - env.HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED, - ]; +function architectureEnvNames(workload: any) { + const forbidden = new Set([ + "HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED", + "HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED", + "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED", + "HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED", + "HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED", + "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED", + ]); + return workload.spec.template.spec.containers[0].env.map((item: any) => item.name).filter((name: string) => forbidden.has(name)); } function snapshot(runtimeDir: string) { diff --git a/scripts/native/hwlab/runtime-gitops-scripts-configmap.mjs b/scripts/native/hwlab/runtime-gitops-scripts-configmap.mjs index b46a8925..7d5311cb 100644 --- a/scripts/native/hwlab/runtime-gitops-scripts-configmap.mjs +++ b/scripts/native/hwlab/runtime-gitops-scripts-configmap.mjs @@ -1,7 +1,7 @@ import { readFileSync } from "node:fs"; import path from "node:path"; -export function runtimeGitopsScriptsConfigMap({ overlay, scriptsDir, namespace, configMapName }) { +export function runtimeGitopsScriptsConfigMap({ overlay, scriptsDir, featureConfigValidatorPath = null, ajvBundlePath = null, namespace, configMapName }) { const data = { "runtime-gitops-overlay.json": `${JSON.stringify({ runtimePath: requiredString(overlay.runtimePath, "overlay.runtimePath"), @@ -9,9 +9,15 @@ export function runtimeGitopsScriptsConfigMap({ overlay, scriptsDir, namespace, codeAgentRuntime: objectOr(overlay.codeAgentRuntime), })}\n`, }; - for (const name of ["runtime-gitops-observability.mjs", "runtime-gitops-postprocess.mjs", "runtime-gitops-verify.mjs"]) { + for (const name of [ + "runtime-gitops-observability.mjs", + "runtime-gitops-postprocess.mjs", + "runtime-gitops-verify.mjs", + ]) { data[name] = readFileSync(path.join(scriptsDir, name), "utf8"); } + data["feature-config-schema-warning.mjs"] = readFileSync(featureConfigValidatorPath ?? path.join(scriptsDir, "feature-config-schema-warning.mjs"), "utf8"); + data["ajv2020.min.js"] = readFileSync(ajvBundlePath ?? path.join(scriptsDir, "ajv2020.min.js"), "utf8"); return { apiVersion: "v1", kind: "ConfigMap", diff --git a/scripts/native/hwlab/runtime-gitops-scripts-configmap.test.ts b/scripts/native/hwlab/runtime-gitops-scripts-configmap.test.ts index 0dc8b141..c252a84a 100644 --- a/scripts/native/hwlab/runtime-gitops-scripts-configmap.test.ts +++ b/scripts/native/hwlab/runtime-gitops-scripts-configmap.test.ts @@ -13,21 +13,22 @@ afterEach(() => { test("preserves codeAgentRuntime in the owning runtime GitOps overlay", () => { const scriptsDir = mkdtempSync(join(tmpdir(), "runtime-gitops-configmap-")); roots.push(scriptsDir); - for (const name of ["runtime-gitops-observability.mjs", "runtime-gitops-postprocess.mjs", "runtime-gitops-verify.mjs"]) { + for (const name of [ + "runtime-gitops-observability.mjs", + "runtime-gitops-postprocess.mjs", + "runtime-gitops-verify.mjs", + "feature-config-schema-warning.mjs", + "ajv2020.min.js", + ]) { writeFileSync(join(scriptsDir, name), `${name}\n`); } const codeAgentRuntime = { enabled: true, kafkaEventBridge: { enabled: true, - features: { - directPublish: false, - liveKafkaSse: false, - kafkaRefreshReplay: false, - transactionalProjector: true, - projectionOutboxRelay: true, - projectionRealtime: true, - }, + transactionalProjector: { heartbeatIntervalMs: 2000 }, + projectionOutboxRelay: { intervalMs: 250 }, + projectionRealtime: { sseHeartbeatMs: 1000 }, }, }; @@ -43,4 +44,6 @@ test("preserves codeAgentRuntime in the owning runtime GitOps overlay", () => { observability: { prometheusOperator: false }, codeAgentRuntime, }); + expect(configMap.data["feature-config-schema-warning.mjs"]).toBe("feature-config-schema-warning.mjs\n"); + expect(configMap.data["ajv2020.min.js"]).toBe("ajv2020.min.js\n"); }); diff --git a/scripts/native/hwlab/runtime-gitops-verify.mjs b/scripts/native/hwlab/runtime-gitops-verify.mjs index 5acdc3c8..88076b18 100644 --- a/scripts/native/hwlab/runtime-gitops-verify.mjs +++ b/scripts/native/hwlab/runtime-gitops-verify.mjs @@ -26,8 +26,6 @@ if (overlay?.observability?.prometheusOperator === false) { } if (overlay?.codeAgentRuntime?.enabled && overlay.codeAgentRuntime.kafkaEventBridge?.enabled) { checks.push("code-agent-runtime-kafka-event-bridge-enabled"); - verifyTransactionalProjectionAuthority(overlay.codeAgentRuntime.kafkaEventBridge); - verifyCodeAgentRuntimeWorkloads(); } console.error(JSON.stringify({ event: "unidesk-runtime-gitops-verify", ok: true, runtimePath, checks })); @@ -43,63 +41,6 @@ function findPrometheusOperatorResources() { return refs; } -function verifyTransactionalProjectionAuthority(kafka) { - const expected = { - directPublish: false, - liveKafkaSse: false, - kafkaRefreshReplay: false, - transactionalProjector: true, - projectionOutboxRelay: true, - projectionRealtime: true, - }; - const actual = Object.fromEntries(Object.keys(expected).map((name) => [name, kafka.features?.[name]])); - const invalid = Object.entries(expected).filter(([name, value]) => actual[name] !== value).map(([name]) => name); - if (invalid.length > 0) fail("kafka-event-bridge-authority-invalid", { invalid, expected, actual }); -} - -function verifyCodeAgentRuntimeWorkloads() { - const expectedByWorkload = { - "hwlab-cloud-api": { - HWLAB_KAFKA_ENABLED: "true", - HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED: "true", - HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false", - HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", - HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", - HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "true", - HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "true", - HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true", - }, - "hwlab-cloud-web": { - HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", - HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", - HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true", - }, - }; - const found = new Set(); - for (const file of listYamlFiles(runtimeDir)) { - const rel = path.relative(repoDir, file); - for (const document of parseStructuredDocuments(readFileSync(file, "utf8"), file)) { - const items = isKubernetesList(document) ? document.items : [document]; - for (const item of items) { - const workloadName = String(item?.metadata?.labels?.["app.kubernetes.io/name"] ?? item?.metadata?.name ?? ""); - const expected = expectedByWorkload[workloadName]; - if (!expected) continue; - const container = item?.spec?.template?.spec?.containers?.find((candidate) => candidate?.name === workloadName); - if (!container) fail("code-agent-runtime-container-missing", { file: rel, workload: workloadName, container: workloadName }); - if (!Array.isArray(container.env)) fail("code-agent-runtime-env-missing", { file: rel, workload: workloadName, container: workloadName }); - const actual = Object.fromEntries(container.env.filter((item) => typeof item?.name === "string").map((item) => [item.name, item.value])); - const invalid = Object.entries(expected) - .filter(([name, value]) => actual[name] !== value) - .map(([name, value]) => ({ name, expected: value, actual: actual[name] ?? null })); - if (invalid.length > 0) fail("code-agent-runtime-capability-env-invalid", { file: rel, workload: workloadName, container: workloadName, invalid }); - found.add(workloadName); - } - } - } - const missing = Object.keys(expectedByWorkload).filter((workloadName) => !found.has(workloadName)); - if (missing.length > 0) fail("code-agent-runtime-workload-missing", { runtimePath, missing }); -} - function splitYamlDocuments(text) { return text.split(/^---[ \t]*(?:#.*)?$/mu).map((doc) => doc.trim()).filter(Boolean); } diff --git a/scripts/native/hwlab/runtime-gitops-verify.test.ts b/scripts/native/hwlab/runtime-gitops-verify.test.ts index bbdf2b75..3f2da472 100644 --- a/scripts/native/hwlab/runtime-gitops-verify.test.ts +++ b/scripts/native/hwlab/runtime-gitops-verify.test.ts @@ -11,101 +11,28 @@ afterEach(() => { while (roots.length > 0) rmSync(roots.pop()!, { recursive: true, force: true }); }); -test("accepts the complete transactional projection capability env", () => { - const fixture = createFixture(); - const result = runVerify(fixture.root); - expect(result.status).toBe(0); - expect(result.stderr).toContain('"code-agent-runtime-kafka-event-bridge-enabled"'); -}); - -test("fails closed when a required workload capability env is missing", () => { - const fixture = createFixture({ omitWebEnv: "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED" }); - const result = runVerify(fixture.root); - expect(result.status).toBe(48); - expect(result.stderr).toContain('"reason":"code-agent-runtime-capability-env-invalid"'); - expect(result.stderr).toContain('"name":"HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"'); -}); - -test("fails closed when a required workload was not matched", () => { - const fixture = createFixture({ omitWebWorkload: true }); - const result = runVerify(fixture.root); - expect(result.status).toBe(48); - expect(result.stderr).toContain('"reason":"code-agent-runtime-workload-missing"'); - expect(result.stderr).toContain('"hwlab-cloud-web"'); -}); - -test("fails closed when the transactional projection authority is incomplete", () => { - const fixture = createFixture({ featureOverrides: { projectionOutboxRelay: false } }); - const result = runVerify(fixture.root); - expect(result.status).toBe(48); - expect(result.stderr).toContain('"reason":"kafka-event-bridge-authority-invalid"'); - expect(result.stderr).toContain('"projectionOutboxRelay"'); -}); - -function createFixture(options: { omitWebEnv?: string; omitWebWorkload?: boolean; featureOverrides?: Record } = {}) { +test("architecture capability env and workload matches are not runtime GitOps gates", () => { const root = mkdtempSync(join(tmpdir(), "runtime-gitops-verify-")); roots.push(root); - const runtimeDir = join(root, "runtime"); - mkdirSync(runtimeDir); - writeFileSync(join(root, "overlay.json"), JSON.stringify(overlay(options.featureOverrides))); - writeFileSync(join(runtimeDir, "cloud-api.yaml"), deploymentYaml("hwlab-cloud-api", cloudApiEnv())); - if (!options.omitWebWorkload) { - writeFileSync(join(runtimeDir, "cloud-web.yaml"), deploymentYaml("hwlab-cloud-web", cloudWebEnv().filter(([name]) => name !== options.omitWebEnv))); - } - return { root }; -} + mkdirSync(join(root, "runtime")); + writeFileSync(join(root, "runtime", "unrelated.yaml"), "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: unrelated\n"); + writeFileSync(join(root, "overlay.json"), JSON.stringify({ + runtimePath: "runtime", + codeAgentRuntime: { + enabled: true, + kafkaEventBridge: { enabled: true }, + }, + })); -function runVerify(root: string) { - return spawnSync(process.execPath, [script], { + const result = spawnSync(process.execPath, [script], { cwd: root, env: { ...process.env, UNIDESK_RUNTIME_GITOPS_OVERLAY_FILE: join(root, "overlay.json") }, encoding: "utf8", }); -} -function overlay(featureOverrides: Record = {}) { - return { - runtimePath: "runtime", - codeAgentRuntime: { - enabled: true, - kafkaEventBridge: { - enabled: true, - features: { - directPublish: false, - liveKafkaSse: false, - kafkaRefreshReplay: false, - transactionalProjector: true, - projectionOutboxRelay: true, - projectionRealtime: true, - ...featureOverrides, - }, - }, - }, - }; -} - -function cloudApiEnv(): Array<[string, string]> { - return [ - ["HWLAB_KAFKA_ENABLED", "true"], - ["HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED", "true"], - ["HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED", "false"], - ["HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED", "false"], - ["HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED", "false"], - ["HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED", "true"], - ["HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED", "true"], - ["HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED", "true"], - ]; -} - -function cloudWebEnv(): Array<[string, string]> { - return [ - ["HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED", "false"], - ["HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED", "false"], - ["HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED", "true"], - ]; -} - -function deploymentYaml(name: string, env: Array<[string, string]>) { - const envYaml = env.map(([envName, value]) => ` - name: ${envName}\n value: "${value}"`).join("\n"); - return `apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: ${name}\n labels:\n app.kubernetes.io/name: ${name}\nspec:\n template:\n spec:\n containers:\n - name: ${name}\n env:\n${envYaml}\n`; -} + expect(result.status).toBe(0); + expect(result.stderr).toContain('"code-agent-runtime-kafka-event-bridge-enabled"'); + expect(result.stderr).not.toContain("capability-env-invalid"); + expect(result.stderr).not.toContain("workload-missing"); + expect(result.stderr).not.toContain("authority-invalid"); +}); diff --git a/scripts/src/agentrun-manifests.ts b/scripts/src/agentrun-manifests.ts index 05b67b80..4424a1a5 100644 --- a/scripts/src/agentrun-manifests.ts +++ b/scripts/src/agentrun-manifests.ts @@ -2,6 +2,8 @@ // Renders AgentRun YAML lane policy into runtime manager environment. import { createHash } from "node:crypto"; import type { AgentRunLaneSpec } from "./agentrun-lanes"; +import { gitRepositoryIdentity, resolveCicdDeliveryAuthority } from "./cicd-delivery-authority"; +import { renderPacFeatureConfigSchemaStage } from "./pac-feature-config-schema-stage"; import { stableJsonSha256 } from "./stable-json"; export interface AgentRunArtifactService { @@ -408,6 +410,19 @@ function agentRunTektonGitopsPublishScript(spec: AgentRunLaneSpec): string { const templateImage = agentRunImageArtifact(spec, { sourceCommit: sourcePlaceholder, envIdentity: envPlaceholder, digest: digestPlaceholder, status: statusPlaceholder }); const templateFiles = renderAgentRunGitopsFiles(spec, { sourceCommit: sourcePlaceholder, image: templateImage }); const templateB64 = Buffer.from(JSON.stringify(templateFiles), "utf8").toString("base64"); + const authority = resolveCicdDeliveryAuthority({ + node: spec.nodeId, + lane: spec.lane, + sourceRepository: gitRepositoryIdentity(spec.source.repository), + sourceBranch: spec.source.branch, + }); + if (authority.kind !== "pac-pr-merge") throw new Error(`AgentRun ${spec.nodeId}/${spec.lane} requires exact PaC consumer authority for feature config validation`); + const featureConfigSchemaStage = renderPacFeatureConfigSchemaStage({ + repositoryId: authority.consumer.repositoryRef, + consumer: authority.consumer.consumerId, + repoDirExpression: '"$root"', + renderedRootExpression: '"$PWD/$(params.gitops-root)"', + }); return [ "#!/bin/sh", "set -eu", @@ -446,6 +461,7 @@ function agentRunTektonGitopsPublishScript(spec: AgentRunLaneSpec): string { " writeFileSync(file.path, content);", "}", "NODE", + featureConfigSchemaStage, "git add source.json \"$(params.artifact-catalog)\" \"$(params.gitops-root)\"", "if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun PaC' commit -m \"deploy: render AgentRun $(params.gitops-branch) from PaC\"; fi", "git remote set-url origin \"$git_auth_url\"", diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 5044acb2..4271d3e5 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -545,22 +545,6 @@ export interface HwlabRuntimeKafkaShadowProducerSpec { export interface HwlabRuntimeKafkaEventBridgeSpec { readonly enabled: boolean; - readonly authority: "transactional-projection"; - readonly features: { - readonly directPublish: boolean; - readonly liveKafkaSse: boolean; - readonly kafkaRefreshReplay: boolean; - readonly transactionalProjector: boolean; - readonly projectionOutboxRelay: boolean; - readonly projectionRealtime: boolean; - }; - readonly refreshReplay?: { - readonly groupIdPrefix: string; - readonly timeoutMs: number; - readonly scanLimit: number; - readonly matchedEventLimit: number; - readonly liveBufferLimit: number; - }; readonly transactionalProjector?: { readonly heartbeatIntervalMs: number; }; @@ -589,7 +573,6 @@ export interface HwlabRuntimeKafkaEventBridgeSpec { readonly agentRunEventTopic: string; readonly hwlabEventTopic: string; readonly clientId: string; - readonly directPublishConsumerGroupId: string; readonly transactionalProjectorConsumerGroupId: string; readonly hwlabEventConsumerGroupId: string; } @@ -1245,27 +1228,8 @@ function codeAgentKafkaShadowProducerConfig(value: unknown, path: string): Hwlab function codeAgentKafkaEventBridgeConfig(value: unknown, path: string): HwlabRuntimeKafkaEventBridgeSpec { const raw = asRecord(value, path); - const features = asRecord(raw.features, `${path}.features`); + if (raw.refreshReplay !== undefined) throw new Error(`${path}.refreshReplay was removed with the fixed projection realtime architecture`); const enabled = booleanField(raw, "enabled", path); - const authority = enumStringField(raw, "authority", path, ["transactional-projection"] as const); - const directPublish = booleanField(features, "directPublish", `${path}.features`); - const liveKafkaSse = booleanField(features, "liveKafkaSse", `${path}.features`); - const kafkaRefreshReplay = booleanField(features, "kafkaRefreshReplay", `${path}.features`); - const transactionalProjector = booleanField(features, "transactionalProjector", `${path}.features`); - const projectionOutboxRelay = booleanField(features, "projectionOutboxRelay", `${path}.features`); - const projectionRealtime = booleanField(features, "projectionRealtime", `${path}.features`); - const refreshReplay = raw.refreshReplay === undefined - ? undefined - : codeAgentKafkaRefreshReplayConfig(raw.refreshReplay, `${path}.refreshReplay`); - if (kafkaRefreshReplay && refreshReplay === undefined) { - throw new Error(`${path}.refreshReplay is required when ${path}.features.kafkaRefreshReplay is true`); - } - if (kafkaRefreshReplay && !liveKafkaSse) { - throw new Error(`${path}.features.liveKafkaSse must be true when ${path}.features.kafkaRefreshReplay is true`); - } - if (!enabled || directPublish || liveKafkaSse || kafkaRefreshReplay || !transactionalProjector || !projectionOutboxRelay || !projectionRealtime) { - throw new Error(`${path}.authority transactional-projection requires enabled transactionalProjector/projectionOutboxRelay/projectionRealtime and disables directPublish/liveKafkaSse/kafkaRefreshReplay`); - } const transactionalProjectorConfig = raw.transactionalProjector === undefined ? undefined : codeAgentKafkaTransactionalProjectorConfig(raw.transactionalProjector, `${path}.transactionalProjector`); @@ -1275,30 +1239,19 @@ function codeAgentKafkaEventBridgeConfig(value: unknown, path: string): HwlabRun const projectionRealtimeConfig = raw.projectionRealtime === undefined ? undefined : codeAgentKafkaProjectionRealtimeConfig(raw.projectionRealtime, `${path}.projectionRealtime`); - if (transactionalProjector && transactionalProjectorConfig === undefined) throw new Error(`${path}.transactionalProjector is required when ${path}.features.transactionalProjector is true`); - if (projectionOutboxRelay && projectionOutboxRelayConfig === undefined) throw new Error(`${path}.projectionOutboxRelay is required when ${path}.features.projectionOutboxRelay is true`); - if (projectionRealtime && projectionRealtimeConfig === undefined) throw new Error(`${path}.projectionRealtime is required when ${path}.features.projectionRealtime is true`); - const directPublishConsumerGroupId = stringField(raw, "directPublishConsumerGroupId", path); + if (transactionalProjectorConfig === undefined) throw new Error(`${path}.transactionalProjector is required by the fixed transactional projection architecture`); + if (projectionOutboxRelayConfig === undefined) throw new Error(`${path}.projectionOutboxRelay is required by the fixed transactional projection architecture`); + if (projectionRealtimeConfig === undefined) throw new Error(`${path}.projectionRealtime is required by the fixed transactional projection architecture`); const transactionalProjectorConsumerGroupId = stringField(raw, "transactionalProjectorConsumerGroupId", path); const hwlabEventConsumerGroupId = stringField(raw, "hwlabEventConsumerGroupId", path); - if (new Set([directPublishConsumerGroupId, transactionalProjectorConsumerGroupId, hwlabEventConsumerGroupId]).size !== 3) { - throw new Error(`${path} consumer group ids must be distinct so enabled capabilities receive independent event streams`); + if (transactionalProjectorConsumerGroupId === hwlabEventConsumerGroupId) { + throw new Error(`${path} projector and realtime consumer group ids must be distinct`); } return { enabled, - authority, - features: { - directPublish, - liveKafkaSse, - kafkaRefreshReplay, - transactionalProjector, - projectionOutboxRelay, - projectionRealtime, - }, - ...(refreshReplay === undefined ? {} : { refreshReplay }), - ...(transactionalProjectorConfig === undefined ? {} : { transactionalProjector: transactionalProjectorConfig }), - ...(projectionOutboxRelayConfig === undefined ? {} : { projectionOutboxRelay: projectionOutboxRelayConfig }), - ...(projectionRealtimeConfig === undefined ? {} : { projectionRealtime: projectionRealtimeConfig }), + transactionalProjector: transactionalProjectorConfig, + projectionOutboxRelay: projectionOutboxRelayConfig, + projectionRealtime: projectionRealtimeConfig, healthThresholds: codeAgentKafkaHealthThresholdsConfig(raw.healthThresholds, `${path}.healthThresholds`), configRef: stringField(raw, "configRef", path), bootstrapServers: stringField(raw, "bootstrapServers", path), @@ -1306,7 +1259,6 @@ function codeAgentKafkaEventBridgeConfig(value: unknown, path: string): HwlabRun agentRunEventTopic: stringField(raw, "agentRunEventTopic", path), hwlabEventTopic: stringField(raw, "hwlabEventTopic", path), clientId: stringField(raw, "clientId", path), - directPublishConsumerGroupId, transactionalProjectorConsumerGroupId, hwlabEventConsumerGroupId, }; @@ -1348,17 +1300,6 @@ function codeAgentKafkaHealthThresholdsConfig(value: unknown, path: string): Hwl }; } -function codeAgentKafkaRefreshReplayConfig(value: unknown, path: string): NonNullable { - const raw = asRecord(value, path); - return { - groupIdPrefix: stringField(raw, "groupIdPrefix", path), - timeoutMs: numberField(raw, "timeoutMs", path), - scanLimit: numberField(raw, "scanLimit", path), - matchedEventLimit: numberField(raw, "matchedEventLimit", path), - liveBufferLimit: numberField(raw, "liveBufferLimit", path), - }; -} - function commandNameField(value: string, path: string): string { if (!/^[A-Za-z0-9._+-]+$/u.test(value)) throw new Error(`${path} must be a simple command name`); return value; diff --git a/scripts/src/hwlab-node-web-observe-runner-realtime-source.test.ts b/scripts/src/hwlab-node-web-observe-runner-realtime-source.test.ts index f95fcbd7..4ef215fe 100644 --- a/scripts/src/hwlab-node-web-observe-runner-realtime-source.test.ts +++ b/scripts/src/hwlab-node-web-observe-runner-realtime-source.test.ts @@ -51,8 +51,8 @@ test("realtime fanout keeps the YAML terminal timeout when durationMs is absent" conditionalEventTypePairs: { "tool-call": { agentrun: "tool_call", hwlab: "tool" } }, expectedKafka: { topics: { stdio: "codex-stdio.raw.v1", agentrun: "agentrun.event.v1", hwlab: "hwlab.event.v1" }, - groups: { directPublish: "direct", transactionalProjector: "projector", liveSse: "live" }, - capabilities: { directPublish: true, liveKafkaSse: true, kafkaRefreshReplay: true }, + groups: { transactionalProjector: "projector", liveSse: "live" }, + capabilities: { transactionalProjector: true, projectionOutboxRelay: true, projectionRealtime: true }, }, }; @@ -194,7 +194,7 @@ test("realtime fanout validates stdio against the scoped AgentRun session lineag assert.equal(expectedStreamSessionId("hwlab", hwlabSessionId), hwlabSessionId); }); -test("realtime fanout validates YAML-owned refresh and legacy live-only connected contracts", () => { +test("realtime fanout validates the fixed transactional replay and live connected contract", () => { const source = nodeWebObserveRunnerRealtimeSource(); const build = new Function(`${source}\nreturn realtimeAssertConnectedContract;`) as () => ( connected: Record, @@ -207,7 +207,7 @@ test("realtime fanout validates YAML-owned refresh and legacy live-only connecte const sessionId = "ses_refresh_contract"; const expectedKafka = { topics: { hwlab: "hwlab.event.v1" }, - capabilities: { directPublish: true, liveKafkaSse: true, kafkaRefreshReplay: true }, + capabilities: { transactionalProjector: true, projectionOutboxRelay: true, projectionRealtime: true }, }; const refreshProfile = { expectedKafka, @@ -249,28 +249,6 @@ test("realtime fanout validates YAML-owned refresh and legacy live-only connecte /replayed count exceeds matched count/u, ); - const liveProfile = { - expectedKafka: { topics: { hwlab: "hwlab.event.v1" }, capabilities: { directPublish: true, liveKafkaSse: true, kafkaRefreshReplay: false } }, - expectedProductSse: { - deliverySemantics: "live-only", - liveOnly: true, - replay: false, - replaySupported: false, - lossPossible: true, - refreshHandoff: false, - replayPriorTraceOnReconnect: false, - }, - }; - assert.doesNotThrow(() => assertConnected({ - realtimeSource: "hwlab.event.v1", - deliverySemantics: "live-only", - liveOnly: true, - replay: false, - replaySupported: false, - lossPossible: true, - filters: { sessionId, traceId: null }, - capabilities: liveProfile.expectedKafka.capabilities, - }, sessionId, "live", liveProfile)); }); test("existing session refresh rejects duplicate or reordered Workbench identities", () => { @@ -326,7 +304,7 @@ test("existing session refresh preserves rejected connected evidence and rejects counts: { matched: 8, replayed: 8, buffered: 1, bufferedDelivered: 1, liveDelivered: 0, deduplicated: 0 }, }; const profile = { - expectedKafka: { topics: { hwlab: "hwlab.event.v1" }, capabilities: { directPublish: true, liveKafkaSse: true, kafkaRefreshReplay: true } }, + expectedKafka: { topics: { hwlab: "hwlab.event.v1" }, capabilities: { transactionalProjector: true, projectionOutboxRelay: true, projectionRealtime: true } }, expectedProductSse: { deliverySemantics: "kafka-retention-then-live", liveOnly: false, replay: true, replaySupported: true, lossPossible: false, refreshHandoff: true }, }; const connected = { diff --git a/scripts/src/hwlab-node-web-observe-runner-realtime-source.ts b/scripts/src/hwlab-node-web-observe-runner-realtime-source.ts index de3a3529..0586236d 100644 --- a/scripts/src/hwlab-node-web-observe-runner-realtime-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-realtime-source.ts @@ -803,11 +803,18 @@ function realtimeFanoutEffectiveProfile(profile, durationMs) { if (!expectedProductSse || typeof expectedProductSse.deliverySemantics !== "string") throw new Error("realtime fanout profile lacks YAML-owned product SSE expectations"); const expectedKafka = profile.expectedKafka && typeof profile.expectedKafka === "object" ? profile.expectedKafka : null; if (!expectedKafka?.topics || !expectedKafka?.groups || !expectedKafka?.capabilities) throw new Error("realtime fanout profile lacks YAML-derived Kafka expectations"); - if (expectedKafka.capabilities.directPublish !== true || expectedKafka.capabilities.liveKafkaSse !== true) { - throw new Error("realtime fanout profile requires YAML-enabled directPublish and liveKafkaSse capabilities"); + if (expectedKafka.capabilities.transactionalProjector !== true + || expectedKafka.capabilities.projectionOutboxRelay !== true + || expectedKafka.capabilities.projectionRealtime !== true) { + throw new Error("realtime fanout profile requires the fixed transactional projection capabilities"); } - if (expectedProductSse.refreshHandoff !== (expectedKafka.capabilities.kafkaRefreshReplay === true)) { - throw new Error("realtime fanout product SSE refreshHandoff differs from the YAML Kafka capability"); + if (expectedProductSse.deliverySemantics !== "kafka-retention-then-live" + || expectedProductSse.liveOnly !== false + || expectedProductSse.replay !== true + || expectedProductSse.replaySupported !== true + || expectedProductSse.lossPossible !== false + || expectedProductSse.refreshHandoff !== true) { + throw new Error("realtime fanout product SSE must use fixed retention replay and live handoff semantics"); } if (expectedProductSse.replayPriorTraceOnReconnect === true && expectedProductSse.replay !== true) { throw new Error("realtime fanout replayPriorTraceOnReconnect requires replay=true"); diff --git a/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts b/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts index e27f8a91..9af4ce6a 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts @@ -791,7 +791,7 @@ export function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName ].join("\n"); } -export function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean): string { +export function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean, featureConfigSchemaStage = ""): string { const gitopsFiles = publishGitops ? sentinelGitopsFiles(state) : []; const filesB64 = Buffer.from(JSON.stringify(gitopsFiles.map((file) => ({ path: file.path, @@ -871,6 +871,7 @@ export function sentinelPublishShell(state: SentinelCicdState, jobName: string, "}", "console.error(JSON.stringify({event:'web-probe-sentinel-gitops-files', fileCount: files.length, valuesRedacted:true}));", "NODE", + ...(featureConfigSchemaStage.length === 0 ? [] : [featureConfigSchemaStage]), " git add .", " file_count=$(git diff --cached --name-only | wc -l | tr -d ' ')", " if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=web-probe-sentinel@unidesk.local -c user.name='UniDesk Web Probe Sentinel' commit -m \"deploy: render web-probe sentinel ${source_commit}\"; fi", diff --git a/scripts/src/hwlab-node/render.ts b/scripts/src/hwlab-node/render.ts index 1dd78a32..f4e31802 100644 --- a/scripts/src/hwlab-node/render.ts +++ b/scripts/src/hwlab-node/render.ts @@ -44,12 +44,15 @@ import { nodeRuntimeDeployYamlOverlayShellScript } from "./deploy-overlay"; import { pipelineProvenanceAnnotationKeys } from "../pipeline-provenance"; import { hwlabNodeDeliveryAuthority, hwlabNodeDeliveryNext } from "./delivery-authority"; import type { CicdDeliveryAuthority } from "../cicd-delivery-authority"; +import { renderPacFeatureConfigSchemaStage } from "../pac-feature-config-schema-stage"; const runtimeGitopsObservabilityNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-observability.mjs"), "utf8").trimEnd(); const runtimeGitopsPipelineGuardNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-pipeline-guard.mjs"), "utf8").trimEnd(); const runtimeGitopsScriptsConfigMapNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-scripts-configmap.mjs"), "utf8").trimEnd(); const runtimeGitopsPostprocessNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-postprocess.mjs"), "utf8").trimEnd(); const runtimeGitopsVerifyNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-verify.mjs"), "utf8").trimEnd(); +const featureConfigSchemaWarningNativeScript = readFileSync(rootPath("scripts/native/cicd/feature-config-schema-warning.mjs"), "utf8").trimEnd(); +const ajv2020BundleScript = readFileSync(rootPath("node_modules/ajv-dist/dist/ajv2020.min.js"), "utf8").trimEnd(); const runtimePipelineProvenanceNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-pipeline-provenance.mjs"), "utf8").trimEnd(); export function nodeRuntimeGitMirrorJobName(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush"): string { @@ -1540,12 +1543,24 @@ export function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec, `--web-endpoint ${shellQuote(spec.publicWebUrl)}`, `--out ${shellQuote(renderDir)}`, ].join(" "), - ...nodeRuntimePipelinePostprocessScript(), + ...nodeRuntimePipelinePostprocessScript(nodeRuntimeFeatureConfigSchemaStage(spec)), ].join("\n"); return { result: runNodeHostScriptAsync(spec, script, timeoutSeconds, `${spec.nodeId.toLowerCase()}-${spec.lane}-render`), renderDir, worktreeDir, location: "node-host" }; } -export function nodeRuntimePipelinePostprocessScript(): string[] { +export function nodeRuntimeFeatureConfigSchemaStage(spec: HwlabRuntimeLaneSpec): string { + const authority = hwlabNodeDeliveryAuthority({ node: spec.nodeId, lane: spec.lane, spec }); + if (authority.kind !== "pac-pr-merge") throw new Error(`HWLAB ${spec.nodeId}/${spec.lane} requires exact PaC consumer authority for feature config validation`); + return renderPacFeatureConfigSchemaStage({ + repositoryId: authority.consumer.repositoryRef, + consumer: authority.consumer.consumerId, + repoDirExpression: '"$PWD"', + renderedRootExpression: JSON.stringify(spec.runtimePath), + ajvBundleExpression: "'/etc/unidesk-cicd-runtime-gitops/ajv2020.min.js'", + }); +} + +export function nodeRuntimePipelinePostprocessScript(featureConfigSchemaStage: string): string[] { return [ "node - \"$render_dir\" \"$overlay_b64\" <<'NODE'", "const fs = require('fs');", @@ -1554,6 +1569,7 @@ export function nodeRuntimePipelinePostprocessScript(): string[] { "const renderDir = process.argv[2];", "const overlay = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));", `const runtimeGitopsObservabilityNativeScript = ${JSON.stringify(runtimeGitopsObservabilityNativeScript)};`, + `const featureConfigSchemaStage = ${JSON.stringify(featureConfigSchemaStage)};`, "const pipelinePath = path.join(renderDir, overlay.tektonDir, 'pipeline.yaml');", "let text = fs.readFileSync(pipelinePath, 'utf8');", "let YAML = null;", @@ -1582,6 +1598,9 @@ export function nodeRuntimePipelinePostprocessScript(): string[] { "echo '{\"event\":\"prepare-source-dependencies\",\"status\":\"not-required\",\"dependency\":\"yaml\",\"manager\":\"inline-service-id-parser\"}' >&2", "ci_timing_emit prepare-source-dependencies succeeded \"$prepare_source_dependencies_started_ms\"`;", "}", + "function featureConfigSchemaValidationScript() {", + " return featureConfigSchemaStage;", + "}", "function validatePrepareSourceDependencyScript(script) {", " const marker = 'NODE_UNIDESK_YAML_DEPENDENCY';", " let offset = 0;", @@ -1838,12 +1857,6 @@ export function nodeRuntimePipelinePostprocessScript(): string[] { "function cloudWebRuntimeEnvEntries() {", " const exposure = overlay.publicExposure;", " const entries = !exposure || !Array.isArray(exposure.extraProxies) ? [] : exposure.extraProxies.filter((proxy) => proxy && proxy.cloudWebEnvName && proxy.publicBaseUrl).map((proxy) => ({ name: String(proxy.cloudWebEnvName), value: String(proxy.publicBaseUrl) }));", - " const kafka = overlay.codeAgentRuntime && overlay.codeAgentRuntime.kafkaEventBridge;", - " if (kafka) {", - " entries.push({ name: 'HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED', value: String(kafka.enabled && kafka.features.liveKafkaSse) });", - " entries.push({ name: 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED', value: String(kafka.enabled && kafka.features.kafkaRefreshReplay) });", - " entries.push({ name: 'HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED', value: String(kafka.enabled && kafka.features.projectionRealtime) });", - " }", " return entries;", "}", "function startupProbeFrom(probe) {", @@ -1941,37 +1954,20 @@ export function nodeRuntimePipelinePostprocessScript(): string[] { " if (codeAgentRuntime.kafkaEventBridge) {", " const kafka = codeAgentRuntime.kafkaEventBridge;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_ENABLED', String(kafka.enabled)) || codeAgentRuntimeChanged;", - " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED', String(kafka.enabled && (kafka.features.directPublish || kafka.features.transactionalProjector))) || codeAgentRuntimeChanged;", + " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED', String(kafka.enabled)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_BOOTSTRAP_SERVERS', String(kafka.bootstrapServers)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_STDIO_TOPIC', String(kafka.stdioTopic)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC', String(kafka.agentRunEventTopic)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_EVENT_TOPIC', String(kafka.hwlabEventTopic)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_CLIENT_ID', String(kafka.clientId)) || codeAgentRuntimeChanged;", - " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID', String(kafka.directPublishConsumerGroupId)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_PROJECTOR_GROUP_ID', String(kafka.transactionalProjectorConsumerGroupId)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID', String(kafka.hwlabEventConsumerGroupId)) || codeAgentRuntimeChanged;", - " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED', String(kafka.enabled && kafka.features.directPublish)) || codeAgentRuntimeChanged;", - " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED', String(kafka.enabled && kafka.features.liveKafkaSse)) || codeAgentRuntimeChanged;", - " const refreshReplayEnabled = kafka.enabled && kafka.features.kafkaRefreshReplay;", - " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED', String(refreshReplayEnabled)) || codeAgentRuntimeChanged;", - " if (refreshReplayEnabled) {", - " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX', String(kafka.refreshReplay.groupIdPrefix)) || codeAgentRuntimeChanged;", - " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS', String(kafka.refreshReplay.timeoutMs)) || codeAgentRuntimeChanged;", - " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT', String(kafka.refreshReplay.scanLimit)) || codeAgentRuntimeChanged;", - " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT', String(kafka.refreshReplay.matchedEventLimit)) || codeAgentRuntimeChanged;", - " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT', String(kafka.refreshReplay.liveBufferLimit)) || codeAgentRuntimeChanged;", - " } else {", - " codeAgentRuntimeChanged = removeEnvValues(container, ['HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX', 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS', 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT', 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT', 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT']) || codeAgentRuntimeChanged;", - " }", - " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED', String(kafka.enabled && kafka.features.transactionalProjector)) || codeAgentRuntimeChanged;", - " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED', String(kafka.enabled && kafka.features.projectionOutboxRelay)) || codeAgentRuntimeChanged;", - " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED', String(kafka.enabled && kafka.features.projectionRealtime)) || codeAgentRuntimeChanged;", - " if (kafka.enabled && kafka.features.transactionalProjector) {", + " if (kafka.enabled) {", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS', String(kafka.transactionalProjector.heartbeatIntervalMs)) || codeAgentRuntimeChanged;", " } else {", " codeAgentRuntimeChanged = removeEnvValues(container, ['HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS']) || codeAgentRuntimeChanged;", " }", - " if (kafka.enabled && kafka.features.projectionOutboxRelay) {", + " if (kafka.enabled) {", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS', String(kafka.projectionOutboxRelay.intervalMs)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE', String(kafka.projectionOutboxRelay.batchSize)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS', String(kafka.projectionOutboxRelay.leaseMs)) || codeAgentRuntimeChanged;", @@ -1980,7 +1976,7 @@ export function nodeRuntimePipelinePostprocessScript(): string[] { " } else {", " codeAgentRuntimeChanged = removeEnvValues(container, ['HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS', 'HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE', 'HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS', 'HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS', 'HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS']) || codeAgentRuntimeChanged;", " }", - " if (kafka.enabled && kafka.features.projectionRealtime) {", + " if (kafka.enabled) {", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE', String(kafka.projectionRealtime.outboxTailBatchSize)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_WORKBENCH_SSE_HEARTBEAT_MS', String(kafka.projectionRealtime.sseHeartbeatMs)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_WORKBENCH_SSE_DRAIN_TIMEOUT_MS', String(kafka.projectionRealtime.sseDrainTimeoutMs)) || codeAgentRuntimeChanged;", @@ -2311,12 +2307,6 @@ export function nodeRuntimePipelinePostprocessScript(): string[] { "function cloudWebRuntimeEnvEntries() {", " const exposure = overlay.publicExposure;", " const entries = !exposure || !Array.isArray(exposure.extraProxies) ? [] : exposure.extraProxies.filter((proxy) => proxy && proxy.cloudWebEnvName && proxy.publicBaseUrl).map((proxy) => ({ name: String(proxy.cloudWebEnvName), value: String(proxy.publicBaseUrl) }));", - " const kafka = overlay.codeAgentRuntime && overlay.codeAgentRuntime.kafkaEventBridge;", - " if (kafka) {", - " entries.push({ name: 'HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED', value: String(kafka.enabled && kafka.features.liveKafkaSse) });", - " entries.push({ name: 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED', value: String(kafka.enabled && kafka.features.kafkaRefreshReplay) });", - " entries.push({ name: 'HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED', value: String(kafka.enabled && kafka.features.projectionRealtime) });", - " }", " return entries;", "}", "function workloadRef(item, file, container) { return { file, kind: item && item.kind, name: item && item.metadata && item.metadata.name, container: container && container.name }; }", @@ -2390,31 +2380,14 @@ export function nodeRuntimePipelinePostprocessScript(): string[] { " if (codeAgentRuntime.kafkaEventBridge) {", " const kafka = codeAgentRuntime.kafkaEventBridge;", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_ENABLED', String(kafka.enabled));", - " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED', String(kafka.enabled && (kafka.features.directPublish || kafka.features.transactionalProjector)));", + " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED', String(kafka.enabled));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_BOOTSTRAP_SERVERS', String(kafka.bootstrapServers));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_STDIO_TOPIC', String(kafka.stdioTopic));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC', String(kafka.agentRunEventTopic));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_EVENT_TOPIC', String(kafka.hwlabEventTopic));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_CLIENT_ID', String(kafka.clientId));", - " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID', String(kafka.directPublishConsumerGroupId));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_PROJECTOR_GROUP_ID', String(kafka.transactionalProjectorConsumerGroupId));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID', String(kafka.hwlabEventConsumerGroupId));", - " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED', String(kafka.enabled && kafka.features.directPublish));", - " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED', String(kafka.enabled && kafka.features.liveKafkaSse));", - " const refreshReplayEnabled = kafka.enabled && kafka.features.kafkaRefreshReplay;", - " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED', String(refreshReplayEnabled));", - " if (refreshReplayEnabled) {", - " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX', String(kafka.refreshReplay.groupIdPrefix));", - " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS', String(kafka.refreshReplay.timeoutMs));", - " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT', String(kafka.refreshReplay.scanLimit));", - " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT', String(kafka.refreshReplay.matchedEventLimit));", - " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT', String(kafka.refreshReplay.liveBufferLimit));", - " } else {", - " for (const envName of ['HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX', 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS', 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT', 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT', 'HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT']) checkCodeAgentRuntimeAbsent(item, file, container, envName);", - " }", - " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED', String(kafka.enabled && kafka.features.transactionalProjector));", - " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED', String(kafka.enabled && kafka.features.projectionOutboxRelay));", - " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED', String(kafka.enabled && kafka.features.projectionRealtime));", " }", " }", " }", @@ -2448,7 +2421,6 @@ export function nodeRuntimePipelinePostprocessScript(): string[] { "if (workloadCheck.wrongCodeAgentRuntimeEnvs.length > 0) fail('code-agent-runtime-env-mismatch', { refs: workloadCheck.wrongCodeAgentRuntimeEnvs.slice(0, 12), count: workloadCheck.wrongCodeAgentRuntimeEnvs.length });", "if (overlay.codeAgentRuntime && overlay.codeAgentRuntime.enabled) checks.push('code-agent-runtime-env');", "if (workloadCheck.wrongCloudWebRuntimeEnvs.length > 0) fail('cloud-web-runtime-env-mismatch', { refs: workloadCheck.wrongCloudWebRuntimeEnvs.slice(0, 12), count: workloadCheck.wrongCloudWebRuntimeEnvs.length });", - "if (cloudWebRuntimeEnvEntries().length > 0) checks.push('cloud-web-runtime-env');", "const pg = overlay.externalPostgres;", "if (pg && pg.serviceName) {", " const access = pg.runtimeAccess || { endpointAddress: pg.endpointAddress, port: pg.port };", @@ -2570,7 +2542,7 @@ export function nodeRuntimePipelinePostprocessScript(): string[] { " });", " result = result.replace(/(node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs[^\\n]*--use-deploy-images[^\\n]*)/g, (match) => {", " if (match.includes('--check')) return match;", - " return `${match}\\n${runtimePathOverlayScript()}`;", + " return `${match}\\n${featureConfigSchemaValidationScript()}\\n${runtimePathOverlayScript()}`;", " });", " result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--out \"\\$render_check_dir\"/g, (match) => match.includes('--gitops-root ') ? match : `${match} --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --node ${shellSingle(overlay.nodeId)} --git-read-url ${shellSingle(overlay.gitReadUrl)} --git-write-url ${shellSingle(overlay.gitWriteUrl)} --runtime-endpoint ${shellSingle(overlay.publicApiUrl)} --web-endpoint ${shellSingle(overlay.publicWebUrl)}`);", " validatePrepareSourceDependencyScript(result);", @@ -2932,6 +2904,8 @@ function runtimeGitopsPipelineGuardScript(): string[] { ...writeRuntimeGitopsNativeScript("runtime-gitops-observability.mjs", runtimeGitopsObservabilityNativeScript, "UNIDESK_RUNTIME_GITOPS_OBSERVABILITY_MJS"), ...writeRuntimeGitopsNativeScript("runtime-gitops-postprocess.mjs", runtimeGitopsPostprocessNativeScript, "UNIDESK_RUNTIME_GITOPS_POSTPROCESS_MJS"), ...writeRuntimeGitopsNativeScript("runtime-gitops-verify.mjs", runtimeGitopsVerifyNativeScript, "UNIDESK_RUNTIME_GITOPS_VERIFY_MJS"), + ...writeRuntimeGitopsNativeScript("feature-config-schema-warning.mjs", featureConfigSchemaWarningNativeScript, "UNIDESK_FEATURE_CONFIG_SCHEMA_WARNING_MJS"), + ...writeRuntimeGitopsNativeScript("ajv2020.min.js", ajv2020BundleScript, "UNIDESK_AJV2020_MIN_JS"), [ "UNIDESK_RUNTIME_GITOPS_OVERLAY_B64=\"$overlay_b64\"", "node \"$runtime_gitops_guard_dir/runtime-gitops-pipeline-guard.mjs\"", diff --git a/scripts/src/hwlab-node/web-probe-observe-actions.test.ts b/scripts/src/hwlab-node/web-probe-observe-actions.test.ts index 39844fb9..11c1906e 100644 --- a/scripts/src/hwlab-node/web-probe-observe-actions.test.ts +++ b/scripts/src/hwlab-node/web-probe-observe-actions.test.ts @@ -18,20 +18,15 @@ test("realtime authority runner contract derives topics, groups, and transaction hwlab: "hwlab.event.v1", }); assert.deepEqual(expectedKafka.groups, { - directPublish: "hwlab-v03-agentrun-event-direct-publish", transactionalProjector: "hwlab-v03-agentrun-event-projector", liveSse: "hwlab-v03-workbench-live-sse", }); assert.deepEqual(expectedKafka.capabilities, { - directPublish: false, - liveKafkaSse: false, - kafkaRefreshReplay: false, transactionalProjector: true, projectionOutboxRelay: true, projectionRealtime: true, }); const kafka = hwlabRuntimeLaneSpecForNode("v03", "NC01").codeAgentRuntime?.kafkaEventBridge; - assert.equal(kafka?.authority, "transactional-projection"); assert.deepEqual(kafka?.transactionalProjector, { heartbeatIntervalMs: 2_000 }); assert.deepEqual(kafka?.projectionOutboxRelay, { intervalMs: 250, diff --git a/scripts/src/hwlab-node/web-probe-observe-actions.ts b/scripts/src/hwlab-node/web-probe-observe-actions.ts index 1bed10cf..2afa7b76 100644 --- a/scripts/src/hwlab-node/web-probe-observe-actions.ts +++ b/scripts/src/hwlab-node/web-probe-observe-actions.ts @@ -23,7 +23,11 @@ export function nodeWebProbeRealtimeFanoutProfiles(spec: HwlabRuntimeLaneSpec): if (Object.keys(profiles).length === 0) return {}; const kafka = spec.codeAgentRuntime?.kafkaEventBridge; if (kafka === undefined) throw new Error("realtimeFanoutProfiles requires codeAgentRuntime.kafkaEventBridge in the owning YAML"); - const capabilities = Object.fromEntries(Object.entries(kafka.features).map(([name, enabled]) => [name, kafka.enabled && enabled])); + const capabilities = { + transactionalProjector: kafka.enabled, + projectionOutboxRelay: kafka.enabled, + projectionRealtime: kafka.enabled, + }; return Object.fromEntries(Object.entries(profiles).map(([id, profile]) => { if (profile.businessEvent !== kafka.hwlabEventTopic) { throw new Error(`realtimeFanoutProfiles.${id}.businessEvent must match codeAgentRuntime.kafkaEventBridge.hwlabEventTopic`); @@ -38,7 +42,6 @@ export function nodeWebProbeRealtimeFanoutProfiles(spec: HwlabRuntimeLaneSpec): hwlab: kafka.hwlabEventTopic, }, groups: { - directPublish: kafka.directPublishConsumerGroupId, transactionalProjector: kafka.transactionalProjectorConsumerGroupId, liveSse: kafka.hwlabEventConsumerGroupId, }, diff --git a/scripts/src/hwlab-node/web-probe.ts b/scripts/src/hwlab-node/web-probe.ts index 18aa5448..7ea2491e 100644 --- a/scripts/src/hwlab-node/web-probe.ts +++ b/scripts/src/hwlab-node/web-probe.ts @@ -1548,7 +1548,7 @@ export function nodeRuntimeCodeAgentRuntimeStatus(spec: HwlabRuntimeLaneSpec, na function nodeRuntimeKafkaAuthorityStatus(spec: HwlabRuntimeLaneSpec, runtime: NonNullable): Record { const kafka = runtime.kafkaEventBridge; - if (kafka === undefined || kafka.enabled === false) return { required: false, authority: kafka?.authority ?? null, valuesPrinted: false }; + if (kafka === undefined || kafka.enabled === false) return { required: false, authority: kafka === undefined ? null : "transactional-projection", valuesPrinted: false }; const healthUrl = `${spec.publicApiUrl.replace(/\/+$/u, "")}/health/ready`; const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", healthUrl], repoRoot, { timeoutMs: 15_000 }); let health: Record = {}; @@ -1575,17 +1575,20 @@ function nodeRuntimeKafkaAuthorityStatus(spec: HwlabRuntimeLaneSpec, runtime: No ].filter((value): value is string => value !== null); return { required: true, - authority: kafka.authority, + authority: "transactional-projection", fixedGroups: { - directPublish: kafka.directPublishConsumerGroupId, transactionalProjector: kafka.transactionalProjectorConsumerGroupId, liveSse: kafka.hwlabEventConsumerGroupId, }, - capabilities: kafka.features, + capabilities: { + transactionalProjector: true, + projectionOutboxRelay: true, + projectionRealtime: true, + }, liveReplay: { - live: kafka.features.projectionRealtime || kafka.features.liveKafkaSse, - replay: kafka.features.projectionRealtime || kafka.features.kafkaRefreshReplay, - cursor: kafka.features.projectionRealtime ? "projection-outbox-seq" : kafka.features.kafkaRefreshReplay ? "kafka-refresh-replay" : "none", + live: true, + replay: true, + cursor: "projection-outbox-seq", }, healthObserved: Object.keys(projector).length > 0, healthStatus: projector.status ?? null, @@ -1691,41 +1694,15 @@ function nodeRuntimeCodeAgentCloudApiEnvStatus(spec: HwlabRuntimeLaneSpec, runti const kafkaEventBridge = runtime.kafkaEventBridge; if (kafkaEventBridge !== undefined) { expectValue("HWLAB_KAFKA_ENABLED", String(kafkaEventBridge.enabled)); - expectValue("HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED", String(kafkaEventBridge.enabled && (kafkaEventBridge.features.directPublish || kafkaEventBridge.features.transactionalProjector))); + expectValue("HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED", String(kafkaEventBridge.enabled)); expectValue("HWLAB_KAFKA_BOOTSTRAP_SERVERS", kafkaEventBridge.bootstrapServers); expectValue("HWLAB_KAFKA_STDIO_TOPIC", kafkaEventBridge.stdioTopic); expectValue("HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC", kafkaEventBridge.agentRunEventTopic); expectValue("HWLAB_KAFKA_EVENT_TOPIC", kafkaEventBridge.hwlabEventTopic); expectValue("HWLAB_KAFKA_CLIENT_ID", kafkaEventBridge.clientId); - expectValue("HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID", kafkaEventBridge.directPublishConsumerGroupId); expectValue("HWLAB_KAFKA_PROJECTOR_GROUP_ID", kafkaEventBridge.transactionalProjectorConsumerGroupId); expectValue("HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID", kafkaEventBridge.hwlabEventConsumerGroupId); - expectValue("HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.directPublish)); - expectValue("HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.liveKafkaSse)); - const refreshReplayEnabled = kafkaEventBridge.enabled && kafkaEventBridge.features.kafkaRefreshReplay; - expectValue("HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED", String(refreshReplayEnabled)); - const refreshReplayEnvNames = [ - "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX", - "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS", - "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT", - "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT", - "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT", - ]; - if (refreshReplayEnabled) { - const refreshReplay = kafkaEventBridge.refreshReplay; - if (refreshReplay === undefined) throw new Error("codeAgentRuntime.kafkaEventBridge.refreshReplay is required when Kafka refresh replay is enabled"); - expectValue("HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX", refreshReplay.groupIdPrefix); - expectValue("HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS", String(refreshReplay.timeoutMs)); - expectValue("HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT", String(refreshReplay.scanLimit)); - expectValue("HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT", String(refreshReplay.matchedEventLimit)); - expectValue("HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT", String(refreshReplay.liveBufferLimit)); - } else { - for (const name of refreshReplayEnvNames) expectAbsent(name); - } - expectValue("HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.transactionalProjector)); - expectValue("HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.projectionOutboxRelay)); - expectValue("HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.projectionRealtime)); - if (kafkaEventBridge.enabled && kafkaEventBridge.features.transactionalProjector) { + if (kafkaEventBridge.enabled) { if (kafkaEventBridge.transactionalProjector === undefined) throw new Error("codeAgentRuntime.kafkaEventBridge.transactionalProjector is required when transactional projector is enabled"); expectValue("HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS", String(kafkaEventBridge.transactionalProjector.heartbeatIntervalMs)); } else { @@ -1738,7 +1715,7 @@ function nodeRuntimeCodeAgentCloudApiEnvStatus(spec: HwlabRuntimeLaneSpec, runti "HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS", "HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS", ]; - if (kafkaEventBridge.enabled && kafkaEventBridge.features.projectionOutboxRelay) { + if (kafkaEventBridge.enabled) { const relay = kafkaEventBridge.projectionOutboxRelay; if (relay === undefined) throw new Error("codeAgentRuntime.kafkaEventBridge.projectionOutboxRelay is required when projection outbox relay is enabled"); expectValue("HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS", String(relay.intervalMs)); @@ -1750,7 +1727,7 @@ function nodeRuntimeCodeAgentCloudApiEnvStatus(spec: HwlabRuntimeLaneSpec, runti for (const name of relayEnvNames) expectAbsent(name); } const projectionRealtimeEnvNames = ["HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE", "HWLAB_WORKBENCH_SSE_HEARTBEAT_MS", "HWLAB_WORKBENCH_SSE_DRAIN_TIMEOUT_MS"]; - if (kafkaEventBridge.enabled && kafkaEventBridge.features.projectionRealtime) { + if (kafkaEventBridge.enabled) { const realtime = kafkaEventBridge.projectionRealtime; if (realtime === undefined) throw new Error("codeAgentRuntime.kafkaEventBridge.projectionRealtime is required when projection realtime is enabled"); expectValue("HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE", String(realtime.outboxTailBatchSize)); @@ -1765,7 +1742,7 @@ function nodeRuntimeCodeAgentCloudApiEnvStatus(spec: HwlabRuntimeLaneSpec, runti deploymentExists: true, deploymentName: "hwlab-cloud-api", containerName: "hwlab-cloud-api", - checkedEnvCount: 9 + (kafkaShadowProducer === undefined ? 0 : 5) + (kafkaEventBridge === undefined ? 0 : 30), + checkedEnvCount: 9 + (kafkaShadowProducer === undefined ? 0 : 5) + (kafkaEventBridge === undefined ? 0 : 22), envCount: envByName.size, mismatches, result: compactCodeAgentDeploymentProbeResult(envProbe), @@ -1774,53 +1751,9 @@ function nodeRuntimeCodeAgentCloudApiEnvStatus(spec: HwlabRuntimeLaneSpec, runti } function nodeRuntimeCodeAgentCloudWebEnvStatus(spec: HwlabRuntimeLaneSpec, runtime: NonNullable): Record { - const kafkaEventBridge = runtime.kafkaEventBridge; - if (kafkaEventBridge === undefined) return { required: false, ready: true, checkedEnvCount: 0, valuesPrinted: false }; - const envProbe = runNodeK3sArgs(spec, [ - "kubectl", - "-n", - spec.runtimeNamespace, - "get", - "deployment", - "hwlab-cloud-web", - "-o", - 'jsonpath={range .spec.template.spec.containers[?(@.name=="hwlab-cloud-web")].env[*]}{.name}{"\\t"}{.value}{"\\t"}{.valueFrom.secretKeyRef.name}{"\\t"}{.valueFrom.secretKeyRef.key}{"\\n"}{end}', - ], 60); - if (envProbe.exitCode !== 0) { - return { - required: true, - ready: false, - deploymentExists: false, - mismatches: [{ name: "hwlab-cloud-web", reason: "deployment-missing" }], - result: compactCodeAgentDeploymentProbeResult(envProbe), - valuesPrinted: false, - }; - } - const envByName = parseCodeAgentEnvProbe(envProbe.stdout); - const mismatches: Array> = []; - const expectValue = (name: string, expected: string): void => { - const item = envByName.get(name); - if (item === undefined) { - mismatches.push({ name, expected, present: false, kind: "value" }); - return; - } - if (item.value !== expected) mismatches.push({ name, expected, value: item.value, present: true, kind: "value" }); - }; - expectValue("HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.liveKafkaSse)); - expectValue("HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.kafkaRefreshReplay)); - expectValue("HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.projectionRealtime)); - return { - required: true, - ready: mismatches.length === 0, - deploymentExists: true, - deploymentName: "hwlab-cloud-web", - containerName: "hwlab-cloud-web", - checkedEnvCount: 3, - envCount: envByName.size, - mismatches, - result: compactCodeAgentDeploymentProbeResult(envProbe), - valuesPrinted: false, - }; + void spec; + void runtime; + return { required: false, ready: true, checkedEnvCount: 0, valuesPrinted: false }; } type CodeAgentEnvProbeRow = { diff --git a/scripts/src/pac-feature-config-schema-stage.test.ts b/scripts/src/pac-feature-config-schema-stage.test.ts new file mode 100644 index 00000000..829f8b7c --- /dev/null +++ b/scripts/src/pac-feature-config-schema-stage.test.ts @@ -0,0 +1,71 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; + +import { resolveAgentRunLaneTarget } from "./agentrun-lanes"; +import { renderAgentRunPipelineManifest } from "./agentrun-manifests"; +import { renderPacFeatureConfigSchemaStage } from "./pac-feature-config-schema-stage"; + +test("shared stage injects owning YAML OTel values for all three PaC repositories", () => { + for (const [repositoryId, consumer] of [ + ["hwlab-nc01-v03", "hwlab-nc01-v03"], + ["agentrun-nc01-v02", "agentrun-nc01-v02"], + ["sentinel-nc01-v03", "sentinel-nc01-v03"], + ] as const) { + const stage = renderPacFeatureConfigSchemaStage({ + repositoryId, + consumer, + repoDirExpression: "'/workspace/source'", + renderedRootExpression: "'/workspace/rendered'", + }); + expect(stage).toContain("feature-config-schema-validation"); + expect(stage).toContain("cicd.feature_config.schema"); + expect(stage).toContain("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT='http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces'"); + expect(stage).toContain("OTEL_TRACES_SAMPLER_ARG='1'"); + expect(stage).toContain("OTEL_EXPORTER_TIMEOUT_MS='300'"); + expect(stage).not.toContain("process.env.webProbeSentinel"); + } +}); + +test("embedded node process failure remains a typed warning with zero shell exit", () => { + const root = mkdtempSync(join(tmpdir(), "feature-config-stage-")); + try { + const bin = join(root, "bin"); + mkdirSync(bin); + writeFileSync(join(bin, "node"), "#!/bin/sh\nexit 71\n", { mode: 0o755 }); + const stage = renderPacFeatureConfigSchemaStage({ + repositoryId: "sentinel-nc01-v03", + consumer: "sentinel-nc01-v03", + repoDirExpression: "'/workspace/source'", + renderedRootExpression: "'/workspace/rendered'", + }); + const result = spawnSync("sh", [], { + input: `set -eu\n${stage}\n`, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? ""}` }, + encoding: "utf8", + }); + expect(result.status).toBe(0); + expect(result.stderr).toContain("feature-config-validator-process-failure"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("AgentRun normal GitOps publish task contains the shared warning-only stage", () => { + const { spec } = resolveAgentRunLaneTarget({ node: "NC01", lane: "nc01-v02" }); + const pipeline = renderAgentRunPipelineManifest(spec) as Record; + const scripts = pipeline.spec.tasks.flatMap((task: Record) => task.taskSpec.steps.map((step: Record) => String(step.script ?? ""))).join("\n"); + expect(scripts).toContain("UNIDESK_PAC_CONSUMER='agentrun-nc01-v02'"); + expect(scripts).toContain("feature-config-schema-validation"); + expect(scripts).toContain("feature-config-validator-process-failure"); + expect(scripts).toContain("OTEL_EXPORTER_TIMEOUT_MS='300'"); +}); + +test("repo-owned schema is Draft 2020-12 with one canonical feature property", () => { + const schema = JSON.parse(readFileSync("config/feature-config.schema.json", "utf8")); + expect(schema.$schema).toBe("https://json-schema.org/draft/2020-12/schema"); + expect(schema.properties.webProbeSentinel["x-unidesk-feature"]).toBe("web-probe-sentinel"); + expect(Object.keys(schema.properties)).toEqual(["webProbeSentinel"]); +}); diff --git a/scripts/src/pac-feature-config-schema-stage.ts b/scripts/src/pac-feature-config-schema-stage.ts new file mode 100644 index 00000000..f5269fcb --- /dev/null +++ b/scripts/src/pac-feature-config-schema-stage.ts @@ -0,0 +1,51 @@ +import { readFileSync } from "node:fs"; + +import { rootPath } from "./config"; +import { loadCicdOtelRuntimeConfig } from "../native/cicd/otel-runtime-config"; + +const validatorSource = readFileSync(rootPath("scripts/native/cicd/feature-config-schema-warning.mjs"), "utf8").trimEnd(); + +export interface PacFeatureConfigSchemaStageOptions { + readonly repositoryId: string; + readonly consumer: string; + readonly repoDirExpression: string; + readonly renderedRootExpression: string; + readonly ajvBundleExpression?: string | null; +} + +export function renderPacFeatureConfigSchemaStage(options: PacFeatureConfigSchemaStageOptions): string { + const otel = loadCicdOtelRuntimeConfig(options.repositoryId); + return [ + "feature_config_schema_started_ms=\"$(date +%s%3N 2>/dev/null || date +%s000)\"", + `export UNIDESK_FEATURE_CONFIG_REPO_DIR=${options.repoDirExpression}`, + `export UNIDESK_FEATURE_CONFIG_RENDERED_ROOT=${options.renderedRootExpression}`, + `export UNIDESK_PAC_CONSUMER=${shellSingle(options.consumer)}`, + `export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=${shellSingle(otel.endpoint)}`, + `export OTEL_SERVICE_NAME=${shellSingle(otel.serviceName)}`, + `export OTEL_TRACES_SAMPLER_ARG=${shellSingle(String(otel.samplingRatio))}`, + `export OTEL_EXPORTER_TIMEOUT_MS=${shellSingle(String(otel.timeoutMs))}`, + ...(options.ajvBundleExpression === undefined || options.ajvBundleExpression === null + ? [] + : [`export UNIDESK_AJV2020_BUNDLE=${options.ajvBundleExpression}`]), + "if ! node --input-type=module <<'NODE_UNIDESK_FEATURE_CONFIG_SCHEMA'", + validatorSource, + "await runFeatureConfigSchemaValidation({", + " repoDir: process.env.UNIDESK_FEATURE_CONFIG_REPO_DIR,", + " renderedRoot: process.env.UNIDESK_FEATURE_CONFIG_RENDERED_ROOT,", + " consumer: process.env.UNIDESK_PAC_CONSUMER,", + "});", + "NODE_UNIDESK_FEATURE_CONFIG_SCHEMA", + "then", + " printf '{\"event\":\"feature-config-schema-validation\",\"warning\":true,\"blocking\":false,\"code\":\"feature-config-validator-process-failure\",\"mutation\":false,\"valuesPrinted\":false}\\n' >&2", + "fi", + "if command -v ci_timing_emit >/dev/null 2>&1; then", + " ci_timing_emit feature-config-schema-validation succeeded \"$feature_config_schema_started_ms\"", + "else", + " printf '{\"event\":\"ci.stage.timing\",\"stage\":\"feature-config-schema-validation\",\"status\":\"succeeded\",\"blocking\":false}\\n' >&2", + "fi", + ].join("\n"); +} + +function shellSingle(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} diff --git a/scripts/src/platform-infra-pac-failure-evidence.test.ts b/scripts/src/platform-infra-pac-failure-evidence.test.ts index a1ececa9..63c28769 100644 --- a/scripts/src/platform-infra-pac-failure-evidence.test.ts +++ b/scripts/src/platform-infra-pac-failure-evidence.test.ts @@ -11,6 +11,9 @@ const require = createRequire(import.meta.url); const { parsePacLogRecords } = require("../native/cicd/pac-status-evaluator.cjs") as { parsePacLogRecords(value: string): Array>; }; +const { extractPacSourceObservation } = require("../native/cicd/pac-status-evaluator.cjs") as { + extractPacSourceObservation(records: Array>, sourceCommit: string): Record; +}; describe("PaC 失败证据合同", () => { test("真实 sentinel TaskRun 日志保留 typed 首断点", () => { @@ -52,6 +55,39 @@ describe("PaC 失败证据合同", () => { expect(compactStatus).toMatchObject({ artifact: { traceId: row.traceId, firstBreak: row.firstBreak, error: row.error } }); }); + test("feature-config warning 在 status、history、debug-step 同构有界投影", () => { + const observation = extractPacSourceObservation([ + { + event: "pac-delivery-plan", + sourceCommitId: "a".repeat(40), + affectedServices: [], + rolloutServices: [], + buildServices: [], + reusedServices: [], + }, + { + event: "feature-config-schema-validation", + warning: true, + blocking: false, + code: "feature-config-input-missing", + schemaPath: "config/feature-config.schema.json", + errorCount: 1, + firstError: "rendered workload candidate snapshot is missing", + mutation: false, + valuesPrinted: false, + }, + ]); + const expected = "feature-config-schema: warning=true blocking=false code=feature-config-input-missing path=config/feature-config.schema.json errors=1 first=rendered workload candidate snapshot is missing"; + const history = renderHistory({ rows: [{ id: "run", sourceObservation: observation }], detailId: "run", config: {}, historyErrors: [], next: {} }).renderedText; + const debug = renderDebugStep({ realRun: { found: true, pipelineRun: {}, artifact: { sourceObservation: observation } }, checks: [], target: {}, next: {} }).renderedText; + const status = renderStatus({ summary: { latestPipelineRun: {}, artifact: {}, diagnostics: {}, sourceObservation: observation }, consumer: {}, deliveryAuthority: {}, config: {}, coverage: [], target: {}, next: {} }).renderedText; + expect(status).toContain(expected); + expect(debug).toContain(expected); + expect(history).toContain("featureConfigCode"); + expect(history).toContain("feature-config-input-missing"); + expect(JSON.stringify(observation)).not.toContain("process.env"); + }); + test("collector 遍历全部 matching TaskRun,terminal record 仍限定合同任务", () => { const remote = readFileSync(resolve(import.meta.dir, "platform-infra-pipelines-as-code-remote.sh"), "utf8"); expect(remote).toContain("for (const item of matching)"); diff --git a/scripts/src/platform-infra-pac-feature-config-projection.test.ts b/scripts/src/platform-infra-pac-feature-config-projection.test.ts new file mode 100644 index 00000000..735b1e60 --- /dev/null +++ b/scripts/src/platform-infra-pac-feature-config-projection.test.ts @@ -0,0 +1,35 @@ +import { expect, test } from "bun:test"; + +import { featureConfigSchemaHistoryFields, renderFeatureConfigSchemaLine } from "./platform-infra-pac-feature-config-projection"; + +test("status and debug share one bounded feature config line", () => { + const observation = { + warning: true, + blocking: false, + code: "feature-config-value-mismatch", + schemaPath: "config/feature-config.schema.json", + errorCount: 2, + firstError: " /webProbeSentinel required must have required property ", + }; + const line = renderFeatureConfigSchemaLine(observation); + expect(line).toBe(" feature-config-schema: warning=true blocking=false code=feature-config-value-mismatch path=config/feature-config.schema.json errors=2 first=/webProbeSentinel required must have required property"); + expect(line.length).toBeLessThan(300); +}); + +test("history uses the same bounded observation fields", () => { + expect(featureConfigSchemaHistoryFields({ + warning: true, + blocking: false, + code: "feature-config-schema-missing", + schemaPath: "config/feature-config.schema.json", + errorCount: 1, + firstError: "schema file is missing", + })).toEqual([ + ["featureConfigWarning", "true"], + ["featureConfigBlocking", "false"], + ["featureConfigCode", "feature-config-schema-missing"], + ["featureConfigSchemaPath", "config/feature-config.schema.json"], + ["featureConfigErrorCount", "1"], + ["featureConfigFirstError", "schema file is missing"], + ]); +}); diff --git a/scripts/src/platform-infra-pac-feature-config-projection.ts b/scripts/src/platform-infra-pac-feature-config-projection.ts new file mode 100644 index 00000000..5532aa34 --- /dev/null +++ b/scripts/src/platform-infra-pac-feature-config-projection.ts @@ -0,0 +1,35 @@ +export function renderFeatureConfigSchemaLine(value: unknown): string { + const featureConfigSchema = record(value); + return ` feature-config-schema: warning=${boolText(featureConfigSchema.warning)} blocking=${boolText(featureConfigSchema.blocking)} code=${stringValue(featureConfigSchema.code)} path=${stringValue(featureConfigSchema.schemaPath)} errors=${stringValue(featureConfigSchema.errorCount)} first=${compactLine(stringValue(featureConfigSchema.firstError))}`; +} + +export function featureConfigSchemaHistoryFields(value: unknown): string[][] { + const featureConfigSchema = record(value); + return [ + ["featureConfigWarning", stringValue(featureConfigSchema.warning)], + ["featureConfigBlocking", stringValue(featureConfigSchema.blocking)], + ["featureConfigCode", stringValue(featureConfigSchema.code)], + ["featureConfigSchemaPath", stringValue(featureConfigSchema.schemaPath)], + ["featureConfigErrorCount", stringValue(featureConfigSchema.errorCount)], + ["featureConfigFirstError", compactLine(stringValue(featureConfigSchema.firstError))], + ]; +} + +function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +function stringValue(value: unknown, fallback = "-"): string { + if (typeof value === "string" && value.length > 0) return value; + if (typeof value === "number" || typeof value === "boolean") return String(value); + return fallback; +} + +function boolText(value: unknown): string { + return value === true ? "true" : "false"; +} + +function compactLine(value: string): string { + const trimmed = value.replace(/\s+/gu, " ").trim(); + return trimmed.length > 0 ? trimmed.slice(0, 120) : "-"; +} diff --git a/scripts/src/platform-infra-pipelines-as-code-source-artifact.test.ts b/scripts/src/platform-infra-pipelines-as-code-source-artifact.test.ts index f70caed0..35316158 100644 --- a/scripts/src/platform-infra-pipelines-as-code-source-artifact.test.ts +++ b/scripts/src/platform-infra-pipelines-as-code-source-artifact.test.ts @@ -7,7 +7,7 @@ import { resolve } from "node:path"; import { rootPath } from "./config"; import { applyNodeRuntimeDeployYamlOverlay, nodeRuntimeDeployYamlOverlayShellScript } from "./hwlab-node/deploy-overlay"; import { hwlabRuntimePipelineProvenance } from "./hwlab-node/pipeline-provenance"; -import { nodeRuntimePipelinePostprocessScript, nodeRuntimePipelineProvenancePostprocessScript } from "./hwlab-node/render"; +import { nodeRuntimeFeatureConfigSchemaStage, nodeRuntimePipelinePostprocessScript, nodeRuntimePipelineProvenancePostprocessScript } from "./hwlab-node/render"; import { nodeRuntimeRenderOverlay } from "./hwlab-node/web-probe"; import { hwlabRuntimeLaneSpecForNode, parseHwlabRuntimePipelineProvenance } from "./hwlab-node-lanes"; import { pipelineProvenanceAnnotationKeys, pipelineProvenanceAnnotations } from "./pipeline-provenance"; @@ -307,7 +307,7 @@ describe("HWLAB YAML-owned Pipeline provenance", () => { "set -eu", `render_dir=${JSON.stringify(temporary)}`, `overlay_b64=${JSON.stringify(overlayBase64)}`, - ...nodeRuntimePipelinePostprocessScript(), + ...nodeRuntimePipelinePostprocessScript(nodeRuntimeFeatureConfigSchemaStage(spec)), ].join("\n"); const result = spawnSync("sh", [], { cwd: rootPath(), @@ -336,14 +336,26 @@ describe("HWLAB YAML-owned Pipeline provenance", () => { expect(scripts.join("\n")).toContain("no-build-no-rollout-plan-gitops-verify"); expect(scripts.join("\n")).not.toContain('"status":"skipped","reason":"no-build-no-rollout-plan"'); expect(scripts.join("\n")).toContain("UNIDESK_RUNTIME_GITOPS_OVERLAY_FILE=/etc/unidesk-cicd-runtime-gitops/runtime-gitops-overlay.json"); + expect(scripts.join("\n")).toContain("export UNIDESK_AJV2020_BUNDLE='/etc/unidesk-cicd-runtime-gitops/ajv2020.min.js'"); + expect(scripts.join("\n")).toContain("feature-config-schema-validation"); + expect(scripts.join("\n")).toContain("cicd.feature_config.schema"); + expect(scripts.join("\n")).toContain("export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT='http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces'"); + expect(scripts.join("\n")).toContain("export OTEL_SERVICE_NAME='unidesk-cicd'"); + expect(scripts.join("\n")).toContain("export OTEL_TRACES_SAMPLER_ARG='1'"); + expect(scripts.join("\n")).toContain("export OTEL_EXPORTER_TIMEOUT_MS='300'"); + expect(scripts.join("\n")).toContain("feature-config-validator-process-failure"); + expect(scripts.join("\n").indexOf("scripts/gitops-render.mjs --use-deploy-images")) + .toBeLessThan(scripts.join("\n").indexOf("feature-config-schema-validation")); expect(scripts.join("\n")).not.toContain("NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS"); expect(scripts.join("\n")).not.toContain("NODE_UNIDESK_RUNTIME_GITOPS_VERIFY"); expect(scripts.join("\n")).not.toContain("UNIDESK_RUNTIME_GITOPS_OVERLAY_B64="); expect(scripts.join("\n")).not.toContain(Buffer.from(JSON.stringify(overlay.observability), "utf8").toString("base64")); const configMap = Bun.YAML.parse(readFileSync(configMapPath, "utf8")) as Record; expect(configMap.metadata.labels["app.kubernetes.io/managed-by"]).toBe("unidesk-host-gitops"); - expect(JSON.parse(configMap.data["runtime-gitops-overlay.json"])).toEqual({ runtimePath: overlay.runtimePath, observability: overlay.observability }); + expect(JSON.parse(configMap.data["runtime-gitops-overlay.json"])).toEqual({ runtimePath: overlay.runtimePath, observability: overlay.observability, codeAgentRuntime: overlay.codeAgentRuntime }); expect(Object.keys(configMap.data).sort()).toEqual([ + "ajv2020.min.js", + "feature-config-schema-warning.mjs", "runtime-gitops-observability.mjs", "runtime-gitops-overlay.json", "runtime-gitops-postprocess.mjs", @@ -352,13 +364,18 @@ describe("HWLAB YAML-owned Pipeline provenance", () => { const hostConfig = Bun.YAML.parse(readFileSync(rootPath("config", "unidesk-host-k8s.yaml"), "utf8")) as Record; const runtimePaths = new Set(hostConfig.delivery.changeDetection.runtimePaths as string[]); for (const inputPath of [ + "bun.lock", "config/hwlab-node-lanes.yaml", + "package.json", + "scripts/native/cicd/feature-config-schema-warning.mjs", "scripts/native/hwlab/runtime-gitops-observability.mjs", "scripts/native/hwlab/runtime-gitops-postprocess.mjs", "scripts/native/hwlab/runtime-gitops-scripts-configmap.mjs", "scripts/native/hwlab/runtime-gitops-verify.mjs", "scripts/src/config.ts", + "scripts/src/hwlab-node/render.ts", "scripts/src/hwlab-node-lanes.ts", + "scripts/src/platform-infra-pipelines-as-code.ts", "scripts/src/yaml-composition.ts", ]) expect(runtimePaths.has(inputPath), inputPath).toBe(true); expect(hostConfig.delivery.gitops.resources).toContainEqual({ @@ -658,14 +675,13 @@ describe("runtime source-commit selection", () => { mode: "remote-pipeline-annotation", maxInlineScriptBytes: 32768, }); - expect([ - spec.codeAgentRuntime?.kafkaEventBridge.features.directPublish, - spec.codeAgentRuntime?.kafkaEventBridge.features.liveKafkaSse, - spec.codeAgentRuntime?.kafkaEventBridge.features.kafkaRefreshReplay, - spec.codeAgentRuntime?.kafkaEventBridge.features.transactionalProjector, - spec.codeAgentRuntime?.kafkaEventBridge.features.projectionOutboxRelay, - spec.codeAgentRuntime?.kafkaEventBridge.features.projectionRealtime, - ]).toEqual([false, false, false, true, true, true]); + expect(spec.codeAgentRuntime?.kafkaEventBridge).toMatchObject({ + enabled: true, + transactionalProjectorConsumerGroupId: "hwlab-v03-agentrun-event-projector", + hwlabEventConsumerGroupId: "hwlab-v03-workbench-live-sse", + }); + expect(spec.codeAgentRuntime?.kafkaEventBridge).not.toHaveProperty("features"); + expect(spec.codeAgentRuntime?.kafkaEventBridge).not.toHaveProperty("refreshReplay"); expect(remoteAnnotations).toEqual(pipelineProvenanceAnnotations(remoteProvenance)); expect(Object.keys(remoteAnnotations).sort()).toEqual(Object.values(pipelineProvenanceAnnotationKeys).sort()); expect(readFileSync(rootPath("scripts", "src", "hwlab-node", "render.ts"), "utf8")).not.toContain("overlay.sourceArtifactProvenance"); diff --git a/scripts/src/platform-infra-pipelines-as-code-source-artifact.ts b/scripts/src/platform-infra-pipelines-as-code-source-artifact.ts index 626d1c4d..95899065 100644 --- a/scripts/src/platform-infra-pipelines-as-code-source-artifact.ts +++ b/scripts/src/platform-infra-pipelines-as-code-source-artifact.ts @@ -9,7 +9,7 @@ import { AGENTRUN_CONFIG_PATH, resolveAgentRunLaneTarget } from "./agentrun-lane import { renderAgentRunPipelineManifest } from "./agentrun-manifests"; import { hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; import { nodeRuntimeGitopsRoot } from "./hwlab-node/cleanup"; -import { nodeRuntimePipelinePostprocessScript } from "./hwlab-node/render"; +import { nodeRuntimeFeatureConfigSchemaStage, nodeRuntimePipelinePostprocessScript } from "./hwlab-node/render"; import { nodeRuntimeRenderOverlay } from "./hwlab-node/web-probe"; import { hwlabRuntimePipelineProvenance } from "./hwlab-node/pipeline-provenance"; import { applyNodeRuntimeDeployYamlOverlay } from "./hwlab-node/deploy-overlay"; @@ -617,7 +617,7 @@ function renderHwlabPipelineFromOwningYaml(spec: HwlabRuntimeLaneSpec, sourceWor "set -eu", `render_dir=${shellQuote(output)}`, `overlay_b64=${shellQuote(overlay)}`, - ...nodeRuntimePipelinePostprocessScript(), + ...nodeRuntimePipelinePostprocessScript(nodeRuntimeFeatureConfigSchemaStage(spec)), ].join("\n"); runChecked("sh", [], temporarySource, 120_000, "HWLAB UniDesk domain postprocess/verify renderer", postprocess); const path = resolve(output, spec.tektonDir, "pipeline.yaml"); diff --git a/scripts/src/platform-infra-pipelines-as-code.ts b/scripts/src/platform-infra-pipelines-as-code.ts index 6d7b6089..8946986c 100644 --- a/scripts/src/platform-infra-pipelines-as-code.ts +++ b/scripts/src/platform-infra-pipelines-as-code.ts @@ -31,6 +31,7 @@ import { pacConsumerRbacDesiredIdentity, renderPacConsumerRbacDesiredFragment } import { runPlatformInfraGiteaCommand } from "./platform-infra-gitea"; import { readGiteaConfig } from "./platform-infra-gitea-config"; import { resolvePacBootstrapMirrorRepository } from "./platform-infra-pipelines-as-code-bootstrap"; +import { featureConfigSchemaHistoryFields, renderFeatureConfigSchemaLine } from "./platform-infra-pac-feature-config-projection"; const configFile = rootPath("config", "platform-infra", "pipelines-as-code.yaml"); const configLabel = "config/platform-infra/pipelines-as-code.yaml"; @@ -2307,6 +2308,7 @@ export function renderStatus(result: Record): RenderedCliResult stringValue(deliveryPlan.reusedServiceCount), stringValue(sourceObservation.reason), ]]), + renderFeatureConfigSchemaLine(sourceObservation.featureConfigSchema), "", "TASKRUN DURATIONS", ...(taskRuns.length === 0 ? ["-"] : table(["TASKRUN", "STATUS", "REASON", "FAILED_STEP", "EXIT", "DURATION_S"], taskRuns.map((item) => { @@ -2524,6 +2526,7 @@ export function renderDebugStep(result: Record): RenderedCliRes stringValue(catalog.digestCount), short(stringValue(artifact.gitopsCommit), 16), ]]), + renderFeatureConfigSchemaLine(record(artifact.sourceObservation).featureConfigSchema), ]), "", "STATUS EVALUATOR FIXTURES", @@ -2946,6 +2949,7 @@ function renderHistoryDetail(row: Record): string[] { ["rolloutServices", stringValue(deliveryPlan.rolloutServiceCount)], ["buildServices", stringValue(deliveryPlan.buildServiceCount)], ["reusedServices", stringValue(deliveryPlan.reusedServiceCount)], + ...featureConfigSchemaHistoryFields(sourceObservation.featureConfigSchema), ]), "", "TERMINAL EVIDENCE",