feat: enable NC01 Kafka event bus config
This commit is contained in:
Codex
@@ -4,12 +4,13 @@ kind: platform-infra-kafka
|
||||
metadata:
|
||||
id: kafka-event-bus
|
||||
owner: unidesk
|
||||
spec: pikasTech/HWLAB#2243
|
||||
spec: pikasTech/HWLAB#2449
|
||||
relatedIssues:
|
||||
- 2243
|
||||
- 2449
|
||||
|
||||
defaults:
|
||||
targetId: D518
|
||||
targetId: NC01
|
||||
switch:
|
||||
enabled: true
|
||||
mode: shadow-produce-only
|
||||
@@ -30,6 +31,13 @@ operator:
|
||||
- kafkausers.kafka.strimzi.io
|
||||
|
||||
targets:
|
||||
- id: NC01
|
||||
route: NC01:k3s
|
||||
namespace: platform-infra
|
||||
role: active
|
||||
enabled: true
|
||||
createNamespace: true
|
||||
storageClassName: local-path
|
||||
- id: D518
|
||||
route: D518:k3s
|
||||
namespace: platform-infra
|
||||
@@ -59,6 +67,24 @@ cluster:
|
||||
mode: deferred-app-integration
|
||||
|
||||
topics:
|
||||
- name: codex-stdio.raw.v1
|
||||
partitions: 1
|
||||
replicas: 1
|
||||
retentionMs: 604800000
|
||||
cleanupPolicy: delete
|
||||
description: AgentRun v0.2 codex stdio raw stdin/stdout/stderr frames for Workbench trace debugging.
|
||||
- name: agentrun.event.v1
|
||||
partitions: 1
|
||||
replicas: 1
|
||||
retentionMs: 604800000
|
||||
cleanupPolicy: delete
|
||||
description: AgentRun v0.2 manager run and command events for HWLAB v0.3 projection.
|
||||
- name: hwlab.event.v1
|
||||
partitions: 1
|
||||
replicas: 1
|
||||
retentionMs: 604800000
|
||||
cleanupPolicy: delete
|
||||
description: HWLAB v0.3 Workbench projection events for SSE passthrough.
|
||||
- name: hwlab.agentrun.command.v1
|
||||
partitions: 1
|
||||
replicas: 1
|
||||
@@ -97,22 +123,34 @@ clients:
|
||||
kafkaUserName: hwlab-v03-cloud-api
|
||||
secretName: hwlab-v03-cloud-api
|
||||
produceTopics:
|
||||
- hwlab.event.v1
|
||||
- hwlab.agentrun.command.v1
|
||||
consumeTopics:
|
||||
- agentrun.event.v1
|
||||
- agentrun.hwlab.event.v1
|
||||
dlqTopics:
|
||||
- agentrun.hwlab.event.dlq.v1
|
||||
- id: agentrun-v02-manager
|
||||
namespace: agentrun-v02
|
||||
serviceAccountName: agentrun-d518-v02-mgr
|
||||
serviceAccountName: agentrun-nc01-v02-mgr
|
||||
kafkaUserName: agentrun-v02-manager
|
||||
secretName: agentrun-v02-manager
|
||||
produceTopics:
|
||||
- agentrun.event.v1
|
||||
- agentrun.hwlab.event.v1
|
||||
consumeTopics:
|
||||
- hwlab.agentrun.command.v1
|
||||
dlqTopics:
|
||||
- hwlab.agentrun.command.dlq.v1
|
||||
- id: agentrun-v02-runner
|
||||
namespace: agentrun-v02
|
||||
serviceAccountName: agentrun-nc01-v02-runner
|
||||
kafkaUserName: agentrun-v02-runner
|
||||
secretName: agentrun-v02-runner
|
||||
produceTopics:
|
||||
- codex-stdio.raw.v1
|
||||
consumeTopics: []
|
||||
dlqTopics: []
|
||||
|
||||
validation:
|
||||
timeoutSeconds: 45
|
||||
@@ -122,5 +160,5 @@ validation:
|
||||
management:
|
||||
defaultTailLimit: 10
|
||||
maxTailLimit: 50
|
||||
defaultShadowTopic: hwlab.agentrun.command.v1
|
||||
defaultShadowTopic: hwlab.event.v1
|
||||
shadowProducerId: unidesk-platform-infra-kafka-cli
|
||||
|
||||
@@ -58,11 +58,11 @@
|
||||
|
||||
## Kafka Event Bus Boundary
|
||||
|
||||
- Kafka for the HWLAB v0.3 / AgentRun v0.2 event-bus POC is a UniDesk-operated platform service in namespace `platform-infra`. It is not owned by `hwlab-v03`, `agentrun-v02`, a per-lane Kafka namespace, or a service repository deployment file.
|
||||
- Kafka for the HWLAB v0.3 / AgentRun v0.2 event-bus POC is a target-scoped UniDesk-operated platform service in namespace `platform-infra`; NC01 is the current validation target for pikasTech/HWLAB#2449. It is not owned by `hwlab-v03`, `agentrun-v02`, a per-lane Kafka namespace, or a service repository deployment file.
|
||||
- The canonical source of truth is `config/platform-infra/kafka.yaml`; target, namespace, Strimzi release URL, cluster name, storage class/size, topic list, client declarations, DLQ names, runtime switch and validation smoke topic must stay in that YAML. Current version numbers and retention values belong only in YAML, not in this reference.
|
||||
- The canonical entrypoint is `bun scripts/cli.ts platform-infra kafka plan|apply|status|validate|topics|groups|offsets|tail|produce --target <node>`; `--node <node>` is an equivalent selector for node-targeted operations. Formal mutation must use that path; raw `kubectl` is bounded diagnosis only.
|
||||
- HWLAB v0.3 and AgentRun v0.2 are client namespaces. They may later consume YAML-declared Kafka bootstrap, user Secret metadata and topic contracts, but app producer/consumer switchover must be a separate HWLAB/AgentRun implementation stage. Runtime readiness alone does not prove Workbench projection, SSE or AgentRun command ingestion has migrated. Shadow produce may write Kafka events for observation only when YAML keeps consumer cutover disabled; it must not replace the current business read path.
|
||||
- The first POC is a single-node KRaft broker for observability, ordering and replay investigation. It improves auditability and smoke coverage, but it is not a production high-availability claim; replication, backup, min ISR and app-side transactional inbox/outbox are separate decisions.
|
||||
- The first target-scoped POC is a single-node KRaft broker for observability, ordering and replay investigation. It improves auditability and smoke coverage, but it is not a production high-availability claim; replication, backup, min ISR and app-side transactional inbox/outbox are separate decisions.
|
||||
- Kafka must stay ClusterIP-only by default. Do not add Ingress, NodePort, LoadBalancer, host networking, public FRP, or browser-facing Kafka access unless a later YAML-controlled platform decision explicitly changes that boundary.
|
||||
|
||||
## LangBot Deployment Boundary
|
||||
|
||||
@@ -182,17 +182,17 @@ export function kafkaHelp(): Record<string, unknown> {
|
||||
command: "platform-infra kafka plan|apply|status|validate|topics|groups|offsets|tail|produce",
|
||||
configTruth: configLabel,
|
||||
usage: [
|
||||
"bun scripts/cli.ts platform-infra kafka plan --target D518",
|
||||
"bun scripts/cli.ts platform-infra kafka plan --node D518",
|
||||
"bun scripts/cli.ts platform-infra kafka apply --target D518 --dry-run",
|
||||
"bun scripts/cli.ts platform-infra kafka apply --target D518 --confirm",
|
||||
"bun scripts/cli.ts platform-infra kafka status --target D518 [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra kafka validate --target D518 [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra kafka topics --node D518",
|
||||
"bun scripts/cli.ts platform-infra kafka groups --node D518",
|
||||
"bun scripts/cli.ts platform-infra kafka offsets --node D518 [--topic hwlab.agentrun.command.v1]",
|
||||
"bun scripts/cli.ts platform-infra kafka tail --node D518 --topic hwlab.agentrun.command.v1 --limit 5",
|
||||
"bun scripts/cli.ts platform-infra kafka produce --node D518 --topic hwlab.agentrun.command.v1 --key <trace-or-session>",
|
||||
"bun scripts/cli.ts platform-infra kafka plan --target NC01",
|
||||
"bun scripts/cli.ts platform-infra kafka plan --node NC01",
|
||||
"bun scripts/cli.ts platform-infra kafka apply --target NC01 --dry-run",
|
||||
"bun scripts/cli.ts platform-infra kafka apply --target NC01 --confirm",
|
||||
"bun scripts/cli.ts platform-infra kafka status --target NC01 [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra kafka validate --target NC01 [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra kafka topics --node NC01",
|
||||
"bun scripts/cli.ts platform-infra kafka groups --node NC01",
|
||||
"bun scripts/cli.ts platform-infra kafka offsets --node NC01 [--topic hwlab.event.v1]",
|
||||
"bun scripts/cli.ts platform-infra kafka tail --node NC01 --topic hwlab.event.v1 --limit 5",
|
||||
"bun scripts/cli.ts platform-infra kafka produce --node NC01 --topic hwlab.event.v1 --key <trace-or-session>",
|
||||
],
|
||||
boundary: "Kafka runtime belongs to platform-infra. P2 supports shadow produce/query only; Kafka consumer cutover stays disabled.",
|
||||
};
|
||||
@@ -327,7 +327,7 @@ function validateConfig(kafka: PlatformKafkaConfig): void {
|
||||
if (!/^https:\/\/github\.com\/strimzi\/strimzi-kafka-operator\/releases\/download\//u.test(kafka.operator.manifestUrl)) {
|
||||
throw new Error(`${configLabel}.operator.manifestUrl must be an official Strimzi GitHub release URL`);
|
||||
}
|
||||
if (kafka.cluster.replicas !== 1) throw new Error(`${configLabel}.cluster.replicas must be 1 for the D518 POC`);
|
||||
if (kafka.cluster.replicas !== 1) throw new Error(`${configLabel}.cluster.replicas must be 1 for the target-scoped Kafka POC`);
|
||||
if (!/^[0-9]+Gi$/u.test(kafka.cluster.storage.size)) throw new Error(`${configLabel}.cluster.storage.size must be a Gi quantity`);
|
||||
if (!kafka.topics.some((topic) => topic.name === kafka.validation.smokeTopic)) throw new Error(`${configLabel}.validation.smokeTopic must reference one of topics[].name`);
|
||||
if (!kafka.topics.some((topic) => topic.name === kafka.management.defaultShadowTopic)) throw new Error(`${configLabel}.management.defaultShadowTopic must reference one of topics[].name`);
|
||||
@@ -967,6 +967,7 @@ def user_summary(item):
|
||||
return {"name": meta_name(item), "ready": condition_ready(item), "secret": status.get("secret")}
|
||||
operator = deployment_ready(load("operator") or {})
|
||||
kafka_obj = load("kafka") or {}
|
||||
nodepool_obj = load("nodepool") or {}
|
||||
topics = [topic_summary(item) for item in items("topics")]
|
||||
users = [user_summary(item) for item in items("users")]
|
||||
for name in expected_topics:
|
||||
@@ -993,7 +994,7 @@ payload = {
|
||||
"tls": "${kafka.cluster.name}-kafka-bootstrap.${target.namespace}.svc.cluster.local:${kafka.cluster.listeners.tls.port}",
|
||||
},
|
||||
"crdsReady": rc("crds") == 0,
|
||||
"nodePoolReady": condition_ready(load("nodepool") or {}),
|
||||
"nodePoolReady": rc("nodepool") == 0 and (condition_ready(nodepool_obj) or condition_ready(kafka_obj)),
|
||||
"topicReady": topic_ready,
|
||||
"userReady": user_ready,
|
||||
"topics": topics,
|
||||
@@ -1485,7 +1486,7 @@ function policyChecks(kafka: PlatformKafkaConfig, target: KafkaTarget, manifest:
|
||||
{ name: "yaml-source-of-truth", ok: true, detail: "Kafka target, namespace, Strimzi version, topics, clients, storage and switch are read from config/platform-infra/kafka.yaml." },
|
||||
{ name: "namespace-is-platform-infra", ok: target.namespace === "platform-infra", detail: "Kafka runtime belongs to platform-infra, not an HWLAB or AgentRun lane namespace." },
|
||||
{ name: "no-public-exposure", ok: !/^\s*type:\s*(NodePort|LoadBalancer)\s*$/mu.test(manifest) && !/^\s*kind:\s*Ingress\s*$/mu.test(manifest), detail: "Kafka POC is ClusterIP/internal only." },
|
||||
{ name: "single-broker-poc", ok: kafka.cluster.replicas === 1, detail: "D518 POC uses one KRaft broker/controller; production HA remains out of scope." },
|
||||
{ name: "single-broker-poc", ok: kafka.cluster.replicas === 1, detail: "The target-scoped POC uses one KRaft broker/controller; production HA remains out of scope." },
|
||||
{ name: "allow-all-network-policy", ok: manifest.includes("kind: NetworkPolicy") && manifest.includes("name: allow-all") && manifest.includes(`namespace: ${target.namespace}`), detail: `NetworkPolicy/allow-all is rendered in ${target.namespace}.` },
|
||||
{ name: "shadow-produce-enabled", ok: kafka.defaults.switch.shadowProduceEnabled === true, detail: "Apps may publish shadow events to Kafka." },
|
||||
{ name: "app-integration-shadow-only", ok: kafka.defaults.switch.appIntegrationEnabled === true && kafka.defaults.switch.mode === "shadow-produce-only", detail: "App integration is limited to shadow production in this stage." },
|
||||
|
||||
@@ -447,16 +447,16 @@ export function platformInfraHelp(): unknown {
|
||||
"bun scripts/cli.ts platform-infra secret-plane apply --target JD01 --confirm",
|
||||
"bun scripts/cli.ts platform-infra secret-plane status --target JD01",
|
||||
"bun scripts/cli.ts platform-infra secret-plane validate --target JD01",
|
||||
"bun scripts/cli.ts platform-infra kafka plan --target D518",
|
||||
"bun scripts/cli.ts platform-infra kafka apply --target D518 --dry-run",
|
||||
"bun scripts/cli.ts platform-infra kafka apply --target D518 --confirm",
|
||||
"bun scripts/cli.ts platform-infra kafka status --target D518",
|
||||
"bun scripts/cli.ts platform-infra kafka validate --target D518",
|
||||
"bun scripts/cli.ts platform-infra kafka topics --node D518",
|
||||
"bun scripts/cli.ts platform-infra kafka groups --node D518",
|
||||
"bun scripts/cli.ts platform-infra kafka offsets --node D518 --topic hwlab.agentrun.command.v1",
|
||||
"bun scripts/cli.ts platform-infra kafka tail --node D518 --topic hwlab.agentrun.command.v1 --limit 5",
|
||||
"bun scripts/cli.ts platform-infra kafka produce --node D518 --topic hwlab.agentrun.command.v1 --key <trace-or-session>",
|
||||
"bun scripts/cli.ts platform-infra kafka plan --target NC01",
|
||||
"bun scripts/cli.ts platform-infra kafka apply --target NC01 --dry-run",
|
||||
"bun scripts/cli.ts platform-infra kafka apply --target NC01 --confirm",
|
||||
"bun scripts/cli.ts platform-infra kafka status --target NC01",
|
||||
"bun scripts/cli.ts platform-infra kafka validate --target NC01",
|
||||
"bun scripts/cli.ts platform-infra kafka topics --node NC01",
|
||||
"bun scripts/cli.ts platform-infra kafka groups --node NC01",
|
||||
"bun scripts/cli.ts platform-infra kafka offsets --node NC01 --topic hwlab.event.v1",
|
||||
"bun scripts/cli.ts platform-infra kafka tail --node NC01 --topic hwlab.event.v1 --limit 5",
|
||||
"bun scripts/cli.ts platform-infra kafka produce --node NC01 --topic hwlab.event.v1 --key <trace-or-session>",
|
||||
"bun scripts/cli.ts platform-infra gitea plan --target JD01",
|
||||
"bun scripts/cli.ts platform-infra gitea apply --target JD01 --dry-run",
|
||||
"bun scripts/cli.ts platform-infra gitea apply --target JD01 --confirm",
|
||||
@@ -470,7 +470,7 @@ export function platformInfraHelp(): unknown {
|
||||
"bun scripts/cli.ts platform-infra pipelines-as-code status --target JD01 --consumer <consumer>",
|
||||
"bun scripts/cli.ts platform-infra pipelines-as-code history --target JD01 --consumer <consumer> --limit 10",
|
||||
],
|
||||
description: "Operate YAML-controlled platform-infra services such as Sub2API, LangBot, n8n, Web Terminal, WeChat archive workflows, OpenTelemetry tracing, the independent target-scoped secret plane, the D518 Kafka event bus, internal Gitea source authority, and Gitea + Pipelines-as-Code closeout for JD01 migrated consumers. Public services use PK01 Caddy/FRP or YAML-declared PK01 Caddy upstreams rather than Kubernetes Ingress, NodePort, or LoadBalancer.",
|
||||
description: "Operate YAML-controlled platform-infra services such as Sub2API, LangBot, n8n, Web Terminal, WeChat archive workflows, OpenTelemetry tracing, the independent target-scoped secret plane, the target-scoped Kafka event bus, internal Gitea source authority, and Gitea + Pipelines-as-Code closeout for JD01 migrated consumers. Public services use PK01 Caddy/FRP or YAML-declared PK01 Caddy upstreams rather than Kubernetes Ingress, NodePort, or LoadBalancer.",
|
||||
target,
|
||||
codexPool: {
|
||||
usage: [
|
||||
|
||||
Reference in New Issue
Block a user