feat: add platform-infra kafka event bus poc

2026-06-28 09:28:38 +00:00
parent 0eb700f07e
commit e8ca6ff009
5 changed files with 1358 additions and 2 deletions
@@ -0,0 +1,118 @@
+version: 1
+kind: platform-infra-kafka
+
+metadata:
+  id: kafka-event-bus
+  owner: unidesk
+  spec: pikasTech/HWLAB#2243
+  relatedIssues:
+    - 2243
+
+defaults:
+  targetId: D518
+  switch:
+    enabled: true
+    mode: runtime-poc
+    appIntegrationEnabled: false
+
+operator:
+  implementation: strimzi
+  version: 1.1.0
+  manifestUrl: https://github.com/strimzi/strimzi-kafka-operator/releases/download/1.1.0/strimzi-cluster-operator-1.1.0.yaml
+  deploymentName: strimzi-cluster-operator
+  serviceAccountName: strimzi-cluster-operator
+  crds:
+    - kafkas.kafka.strimzi.io
+    - kafkanodepools.kafka.strimzi.io
+    - kafkatopics.kafka.strimzi.io
+    - kafkausers.kafka.strimzi.io
+
+targets:
+  - id: D518
+    route: D518:k3s
+    namespace: platform-infra
+    role: active
+    enabled: true
+    createNamespace: true
+    storageClassName: local-path
+
+cluster:
+  name: platform-infra-kafka
+  nodePoolName: dual-role
+  kafkaVersion: 4.3.0
+  metadataVersion: 4.3-IV0
+  replicas: 1
+  storage:
+    size: 4Gi
+    deleteClaim: false
+  listeners:
+    plain:
+      enabled: true
+      port: 9092
+    tls:
+      enabled: true
+      port: 9093
+  authorization:
+    enabled: false
+    mode: deferred-app-integration
+
+topics:
+  - name: hwlab.agentrun.command.v1
+    partitions: 1
+    replicas: 1
+    retentionMs: 604800000
+    cleanupPolicy: delete
+    description: HWLAB v0.3 admitted user commands for AgentRun v0.2.
+  - name: agentrun.hwlab.event.v1
+    partitions: 1
+    replicas: 1
+    retentionMs: 604800000
+    cleanupPolicy: delete
+    description: AgentRun v0.2 run/command events for HWLAB v0.3 ingestion.
+  - name: hwlab.agentrun.command.dlq.v1
+    partitions: 1
+    replicas: 1
+    retentionMs: 1209600000
+    cleanupPolicy: delete
+    description: Dead-letter topic for HWLAB to AgentRun command delivery.
+  - name: agentrun.hwlab.event.dlq.v1
+    partitions: 1
+    replicas: 1
+    retentionMs: 1209600000
+    cleanupPolicy: delete
+    description: Dead-letter topic for AgentRun to HWLAB event delivery.
+  - name: platform-infra.kafka.smoke.v1
+    partitions: 1
+    replicas: 1
+    retentionMs: 86400000
+    cleanupPolicy: delete
+    description: Platform-infra Kafka broker smoke topic.
+
+clients:
+  - id: hwlab-v03-cloud-api
+    namespace: hwlab-v03
+    serviceAccountName: hwlab-cloud-api
+    kafkaUserName: hwlab-v03-cloud-api
+    secretName: hwlab-v03-cloud-api
+    produceTopics:
+      - hwlab.agentrun.command.v1
+    consumeTopics:
+      - agentrun.hwlab.event.v1
+    dlqTopics:
+      - agentrun.hwlab.event.dlq.v1
+  - id: agentrun-v02-manager
+    namespace: agentrun-v02
+    serviceAccountName: agentrun-manager
+    kafkaUserName: agentrun-v02-manager
+    secretName: agentrun-v02-manager
+    produceTopics:
+      - agentrun.hwlab.event.v1
+    consumeTopics:
+      - hwlab.agentrun.command.v1
+    dlqTopics:
+      - hwlab.agentrun.command.dlq.v1
+
+validation:
+  timeoutSeconds: 45
+  pollSeconds: 3
+  smokeTopic: platform-infra.kafka.smoke.v1
@@ -33,6 +33,15 @@
 - External platform PostgreSQL endpoints for Sub2API are produced by the platform DB YAML and its `platform-db postgres` CLI. Cross-node Sub2API consumers connect directly to that endpoint; the master server is not a PostgreSQL data-plane relay. DNS aliases are optional when the exported `DATABASE_URL` uses a reachable IP with `sslmode=require`; current PK01-specific rules live in `docs/reference/pk01.md`.
 - Sub2API account sentinel, public exposure, and HTTPS egress proxy are target-scoped YAML decisions. The active target may run them when YAML enables them; the standby G14 target must stay deployed but inactive until YAML promotion. `sentinel.enabledOnTargets` is the authority for where Codex-pool sentinel image, CronJob, Secret and state resources are expected; disabled targets should report sentinel validation as skipped instead of failing on missing runtime sentinel objects. Do not create a second sentinel, FRP client, public management surface, or edge proxy by hand; enable or move those resources only through the target YAML and the `platform-infra sub2api` / `codex-pool --target` CLI paths.

+## Kafka Event Bus Boundary
+
+- Kafka for the HWLAB v0.3 / AgentRun v0.2 event-bus POC is a UniDesk-operated platform service in namespace `platform-infra`. It is not owned by `hwlab-v03`, `agentrun-v02`, a per-lane Kafka namespace, or a service repository deployment file.
+- The canonical source of truth is `config/platform-infra/kafka.yaml`; target, namespace, Strimzi release URL, cluster name, storage class/size, topic list, client declarations, DLQ names, runtime switch and validation smoke topic must stay in that YAML. Current version numbers and retention values belong only in YAML, not in this reference.
+- The canonical entrypoint is `bun scripts/cli.ts platform-infra kafka plan|apply|status|validate --target D518`. Formal mutation must use that path; raw `kubectl` is bounded diagnosis only.
+- HWLAB v0.3 and AgentRun v0.2 are client namespaces. They may later consume YAML-declared Kafka bootstrap, user Secret metadata and topic contracts, but app producer/consumer switchover must be a separate HWLAB/AgentRun implementation stage. Runtime readiness alone does not prove Workbench projection, SSE or AgentRun command ingestion has migrated.
+- The first POC is a single-node KRaft broker for observability, ordering and replay investigation. It improves auditability and smoke coverage, but it is not a production high-availability claim; replication, backup, min ISR and app-side transactional inbox/outbox are separate decisions.
+- Kafka must stay ClusterIP-only by default. Do not add Ingress, NodePort, LoadBalancer, host networking, public FRP, or browser-facing Kafka access unless a later YAML-controlled platform decision explicitly changes that boundary.
+
 ## LangBot Deployment Boundary

 - LangBot is a UniDesk-operated public platform service in namespace `platform-infra`. The canonical entrypoint is `bun scripts/cli.ts platform-infra langbot plan|apply|status|logs|validate|bootstrap-api-key|query`; G14 is the default runtime target.
@@ -314,7 +314,7 @@ export interface ManagedResourceCleanupPlan {
 export function platformInfraHelp(): unknown {
  const target = sub2ApiHelpTargetSummary();
  return {
-    command: "platform-infra sub2api|langbot|n8n|wechat-archive|observability|secret-plane ...",
+    command: "platform-infra sub2api|langbot|n8n|wechat-archive|observability|secret-plane|kafka ...",
    output: "json",
    usage: [
      "bun scripts/cli.ts platform-infra sub2api plan [--target G14|D601]",
@@ -365,8 +365,13 @@ export function platformInfraHelp(): unknown {
      "bun scripts/cli.ts platform-infra secret-plane apply --target D518 --confirm",
      "bun scripts/cli.ts platform-infra secret-plane status --target D518",
      "bun scripts/cli.ts platform-infra secret-plane validate --target D518",
+      "bun scripts/cli.ts platform-infra kafka plan --target D518",
+      "bun scripts/cli.ts platform-infra kafka apply --target D518 --dry-run",
+      "bun scripts/cli.ts platform-infra kafka apply --target D518 --confirm",
+      "bun scripts/cli.ts platform-infra kafka status --target D518",
+      "bun scripts/cli.ts platform-infra kafka validate --target D518",
    ],
-    description: "Operate YAML-controlled platform-infra services such as Sub2API, LangBot, n8n, WeChat archive workflows, OpenTelemetry tracing and the independent target-scoped secret plane. Public services use PK01 Caddy+FRP rather than Kubernetes Ingress, NodePort, or LoadBalancer.",
+    description: "Operate YAML-controlled platform-infra services such as Sub2API, LangBot, n8n, WeChat archive workflows, OpenTelemetry tracing, the independent target-scoped secret plane, and the D518 Kafka event bus. Public services use PK01 Caddy+FRP rather than Kubernetes Ingress, NodePort, or LoadBalancer.",
    target,
    codexPool: {
      usage: [
@@ -60,6 +60,10 @@ export async function runPlatformInfraCommand(config: UniDeskConfig, args: strin
    const { runSecretPlaneCommand } = await import("../platform-infra-secret-plane");
    return await runSecretPlaneCommand(config, args.slice(1));
  }
+  if (target === "kafka") {
+    const { runPlatformInfraKafkaCommand } = await import("../platform-infra-kafka");
+    return await runPlatformInfraKafkaCommand(config, args.slice(1));
+  }
  if (target !== "sub2api") return unsupported(args);
  if (action === "plan" || action === undefined) {
    const planArgs = args.slice(2);