mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enable d518 kafka shadow producers

Browse Source
This commit is contained in:
Codex
2026-06-28 11:51:34 +00:00
parent 83f898b11f
commit fb7bbc1b63
8 changed files with 75 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/agentrun.yaml
+5
View File
@@ -605,6 +605,11 @@ controlPlane:
AGENTRUN_MANAGER_RECONCILER_BATCH_SIZE: "20"
AGENTRUN_MANAGER_RECONCILER_ENABLED: "true"
AGENTRUN_MANAGER_RECONCILER_INTERVAL_MS: "30000"
AGENTRUN_KAFKA_SHADOW_PRODUCE_ENABLED: "true"
AGENTRUN_KAFKA_SHADOW_CONSUME_ENABLED: "false"
AGENTRUN_KAFKA_BOOTSTRAP_SERVERS: platform-infra-kafka-kafka-bootstrap.platform-infra.svc.cluster.local:9092
AGENTRUN_KAFKA_EVENT_TOPIC: agentrun.hwlab.event.v1
AGENTRUN_KAFKA_CLIENT_ID: agentrun-v02-manager
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces
OTEL_SERVICE_NAME: agentrun-manager
UNIDESK_NODE_ID: D518
config/hwlab-node-lanes.yaml
+8
View File
@@ -710,6 +710,14 @@ lanes:
providerIdFrom: runtimeNodeId
defaultProviderProfile: fake-echo
codexStdioSupervisor: repo-owned
kafkaShadowProducer:
enabled: true
mode: shadow-produce-only
consumeEnabled: false
configRef: config/platform-infra/kafka.yaml#clients.hwlab-v03-cloud-api
bootstrapServers: platform-infra-kafka-kafka-bootstrap.platform-infra.svc.cluster.local:9092
commandTopic: hwlab.agentrun.command.v1
clientId: hwlab-v03-cloud-api
publicExposure:
mode: pk01-caddy-frp
publicBaseUrl: https://hwlab.pikapython.com
config/platform-infra/kafka.yaml
+2 -2
View File
@@ -13,7 +13,7 @@ defaults:
switch:
enabled: true
mode: shadow-produce-only
appIntegrationEnabled: false
appIntegrationEnabled: true
shadowProduceEnabled: true
shadowConsumeEnabled: false
@@ -104,7 +104,7 @@ clients:
- agentrun.hwlab.event.dlq.v1
- id: agentrun-v02-manager
namespace: agentrun-v02
serviceAccountName: agentrun-manager
serviceAccountName: agentrun-d518-v02-mgr
kafkaUserName: agentrun-v02-manager
secretName: agentrun-v02-manager
produceTopics:
scripts/src/hwlab-node-lanes.ts
+30
View File
@@ -322,6 +322,16 @@ export interface HwlabRuntimeCodeAgentProviderSpec {
readonly opencodeSourceKey?: string;
}
export interface HwlabRuntimeKafkaShadowProducerSpec {
readonly enabled: boolean;
readonly mode: "shadow-produce-only";
readonly consumeEnabled: false;
readonly configRef: string;
readonly bootstrapServers: string;
readonly commandTopic: string;
readonly clientId: string;
}
export interface HwlabRuntimeCodeAgentRuntimeSpec {
readonly enabled: boolean;
readonly adapter: string;
@@ -334,6 +344,7 @@ export interface HwlabRuntimeCodeAgentRuntimeSpec {
readonly providerIdFrom: "runtimeNodeId";
readonly defaultProviderProfile: string;
readonly codexStdioSupervisor: "repo-owned";
readonly kafkaShadowProducer?: HwlabRuntimeKafkaShadowProducerSpec;
}
export interface HwlabRuntimeSourceWorkspaceSpec {
@@ -795,6 +806,25 @@ function codeAgentRuntimeConfig(value: unknown, path: string): HwlabRuntimeCodeA
providerIdFrom: enumStringField(raw, "providerIdFrom", path, ["runtimeNodeId"]),
defaultProviderProfile: stringField(raw, "defaultProviderProfile", path),
codexStdioSupervisor: enumStringField(raw, "codexStdioSupervisor", path, ["repo-owned"]),
...(raw.kafkaShadowProducer === undefined
? {}
: { kafkaShadowProducer: codeAgentKafkaShadowProducerConfig(raw.kafkaShadowProducer, `${path}.kafkaShadowProducer`) }),
};
}
function codeAgentKafkaShadowProducerConfig(value: unknown, path: string): HwlabRuntimeKafkaShadowProducerSpec {
const raw = asRecord(value, path);
const mode = enumStringField(raw, "mode", path, ["shadow-produce-only"]);
const consumeEnabled = booleanField(raw, "consumeEnabled", path);
if (consumeEnabled !== false) throw new Error(`${path}.consumeEnabled must be false during shadow-produce-only stage`);
return {
enabled: booleanField(raw, "enabled", path),
mode,
consumeEnabled,
configRef: stringField(raw, "configRef", path),
bootstrapServers: stringField(raw, "bootstrapServers", path),
commandTopic: stringField(raw, "commandTopic", path),
clientId: stringField(raw, "clientId", path),
};
}
scripts/src/hwlab-node/plan.ts
+1
View File
@@ -158,6 +158,7 @@ export function nodeRuntimeExpected(spec: HwlabRuntimeLaneSpec): Record<string,
providerId: spec.nodeId,
defaultProviderProfile: spec.codeAgentRuntime.defaultProviderProfile,
codexStdioSupervisor: spec.codeAgentRuntime.codexStdioSupervisor,
kafkaShadowProducer: spec.codeAgentRuntime.kafkaShadowProducer ?? null,
valuesPrinted: false,
},
sourceWorkspace: spec.sourceWorkspace === undefined ? null : {
scripts/src/hwlab-node/render.ts
+16
View File
@@ -1570,6 +1570,14 @@ export function nodeRuntimePipelinePostprocessScript(): string[] {
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID', String(codeAgentRuntime.providerId)) || codeAgentRuntimeChanged;",
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE', String(codeAgentRuntime.defaultProviderProfile)) || codeAgentRuntimeChanged;",
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR', String(codeAgentRuntime.codexStdioSupervisor)) || codeAgentRuntimeChanged;",
" if (codeAgentRuntime.kafkaShadowProducer) {",
" const kafka = codeAgentRuntime.kafkaShadowProducer;",
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_SHADOW_PRODUCE_ENABLED', String(kafka.enabled)) || codeAgentRuntimeChanged;",
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_SHADOW_CONSUME_ENABLED', String(kafka.consumeEnabled)) || codeAgentRuntimeChanged;",
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_BOOTSTRAP_SERVERS', String(kafka.bootstrapServers)) || codeAgentRuntimeChanged;",
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_COMMAND_TOPIC', String(kafka.commandTopic)) || codeAgentRuntimeChanged;",
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_CLIENT_ID', String(kafka.clientId)) || codeAgentRuntimeChanged;",
" }",
" }",
" }",
" }",
@@ -1895,6 +1903,14 @@ export function nodeRuntimePipelinePostprocessScript(): string[] {
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID', String(codeAgentRuntime.providerId));",
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE', String(codeAgentRuntime.defaultProviderProfile));",
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR', String(codeAgentRuntime.codexStdioSupervisor));",
" if (codeAgentRuntime.kafkaShadowProducer) {",
" const kafka = codeAgentRuntime.kafkaShadowProducer;",
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_SHADOW_PRODUCE_ENABLED', String(kafka.enabled));",
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_SHADOW_CONSUME_ENABLED', String(kafka.consumeEnabled));",
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_BOOTSTRAP_SERVERS', String(kafka.bootstrapServers));",
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_COMMAND_TOPIC', String(kafka.commandTopic));",
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_CLIENT_ID', String(kafka.clientId));",
" }",
" }",
" }",
" if (Array.isArray(podSpec.volumes) && podSpec.volumes.some((volume) => volume && volume.name === 'hwlab-metrics-sidecar')) metricsRefs.push(workloadRef(item, file, { name: 'volume/hwlab-metrics-sidecar' }));",
scripts/src/hwlab-node/web-probe.ts
+10 -1
View File
@@ -108,6 +108,7 @@ export function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record<str
providerId: spec.nodeId,
defaultProviderProfile: spec.codeAgentRuntime.defaultProviderProfile,
codexStdioSupervisor: spec.codeAgentRuntime.codexStdioSupervisor,
kafkaShadowProducer: spec.codeAgentRuntime.kafkaShadowProducer ?? null,
valuesPrinted: false,
},
runtimeImageRewrites: spec.runtimeImageRewrites,
@@ -1441,12 +1442,20 @@ function nodeRuntimeCodeAgentCloudApiEnvStatus(spec: HwlabRuntimeLaneSpec, runti
expectValue("HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID", spec.nodeId);
expectValue("HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE", runtime.defaultProviderProfile);
expectValue("HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR", runtime.codexStdioSupervisor);
const kafkaShadowProducer = runtime.kafkaShadowProducer;
if (kafkaShadowProducer !== undefined) {
expectValue("HWLAB_KAFKA_SHADOW_PRODUCE_ENABLED", String(kafkaShadowProducer.enabled));
expectValue("HWLAB_KAFKA_SHADOW_CONSUME_ENABLED", String(kafkaShadowProducer.consumeEnabled));
expectValue("HWLAB_KAFKA_BOOTSTRAP_SERVERS", kafkaShadowProducer.bootstrapServers);
expectValue("HWLAB_KAFKA_COMMAND_TOPIC", kafkaShadowProducer.commandTopic);
expectValue("HWLAB_KAFKA_CLIENT_ID", kafkaShadowProducer.clientId);
}
return {
ready: mismatches.length === 0,
deploymentExists: true,
deploymentName: "hwlab-cloud-api",
containerName: container?.name ?? null,
checkedEnvCount: 8,
checkedEnvCount: kafkaShadowProducer === undefined ? 9 : 14,
mismatches,
result: compactCodeAgentDeploymentProbeResult(deployment),
valuesPrinted: false,
scripts/src/platform-infra-kafka.ts
+3 -2
View File
@@ -1487,8 +1487,9 @@ function policyChecks(kafka: PlatformKafkaConfig, target: KafkaTarget, manifest:
{ name: "no-public-exposure", ok: !/^\s*type:\s*(NodePort|LoadBalancer)\s*$/mu.test(manifest) && !/^\s*kind:\s*Ingress\s*$/mu.test(manifest), detail: "Kafka POC is ClusterIP/internal only." },
{ name: "single-broker-poc", ok: kafka.cluster.replicas === 1, detail: "D518 POC uses one KRaft broker/controller; production HA remains out of scope." },
{ name: "allow-all-network-policy", ok: manifest.includes("kind: NetworkPolicy") && manifest.includes("name: allow-all") && manifest.includes(`namespace: ${target.namespace}`), detail: `NetworkPolicy/allow-all is rendered in ${target.namespace}.` },
{ name: "shadow-produce-enabled", ok: kafka.defaults.switch.shadowProduceEnabled === true, detail: "P2 may publish shadow events to Kafka." },
{ name: "shadow-consume-disabled", ok: kafka.defaults.switch.shadowConsumeEnabled === false && kafka.defaults.switch.appIntegrationEnabled === false, detail: "P2 does not enable Kafka consumer cutover or app integration." },
{ name: "shadow-produce-enabled", ok: kafka.defaults.switch.shadowProduceEnabled === true, detail: "Apps may publish shadow events to Kafka." },
{ name: "app-integration-shadow-only", ok: kafka.defaults.switch.appIntegrationEnabled === true && kafka.defaults.switch.mode === "shadow-produce-only", detail: "App integration is limited to shadow production in this stage." },
{ name: "shadow-consume-disabled", ok: kafka.defaults.switch.shadowConsumeEnabled === false, detail: "Kafka consumer cutover remains disabled." },
];
}