fix: 切换 HWLAB 投影实时权威
This commit is contained in:
Codex
@@ -753,6 +753,8 @@ export function summarizeNodeRuntimeControlPlaneStatus(
|
||||
const pipelineRunDiagnostics = record(status.pipelineRunDiagnostics);
|
||||
const argo = record(status.argo);
|
||||
const runtime = record(status.runtime);
|
||||
const codeAgentRuntime = record(runtime.codeAgentRuntime);
|
||||
const kafkaAuthority = record(codeAgentRuntime.kafkaAuthority);
|
||||
const publicProbes = record(status.publicProbes);
|
||||
const gitMirror = record(status.gitMirror);
|
||||
const gitMirrorCompact = record(gitMirror.compact);
|
||||
@@ -822,8 +824,25 @@ export function summarizeNodeRuntimeControlPlaneStatus(
|
||||
externalPostgresReady: runtime.externalPostgresBridge === undefined && runtime.externalPostgresSecrets === undefined
|
||||
? null
|
||||
: record(runtime.externalPostgresBridge).ready === true && record(runtime.externalPostgresSecrets).ready === true,
|
||||
codeAgentRuntimeReady: record(runtime.codeAgentRuntime).required === true ? record(runtime.codeAgentRuntime).ready === true : null,
|
||||
codeAgentRuntimeReason: typeof record(runtime.codeAgentRuntime).degradedReason === "string" ? record(runtime.codeAgentRuntime).degradedReason : null,
|
||||
codeAgentRuntimeReady: codeAgentRuntime.required === true ? codeAgentRuntime.ready === true : null,
|
||||
codeAgentRuntimeReason: typeof codeAgentRuntime.degradedReason === "string" ? codeAgentRuntime.degradedReason : null,
|
||||
kafkaAuthority: Object.keys(kafkaAuthority).length === 0 ? null : {
|
||||
authority: kafkaAuthority.authority ?? null,
|
||||
fixedGroups: kafkaAuthority.fixedGroups ?? null,
|
||||
capabilities: kafkaAuthority.capabilities ?? null,
|
||||
liveReplay: kafkaAuthority.liveReplay ?? null,
|
||||
healthObserved: kafkaAuthority.healthObserved === true,
|
||||
healthStatus: kafkaAuthority.healthStatus ?? null,
|
||||
consumerLag: kafkaAuthority.consumerLag ?? null,
|
||||
backlogCount: kafkaAuthority.backlogCount ?? null,
|
||||
retryCount: kafkaAuthority.retryCount ?? null,
|
||||
failedInboxCount: kafkaAuthority.failedInboxCount ?? null,
|
||||
dlqCount: kafkaAuthority.dlqCount ?? null,
|
||||
thresholds: kafkaAuthority.thresholds ?? null,
|
||||
warning: kafkaAuthority.warning === true,
|
||||
warningReasons: kafkaAuthority.warningReasons ?? [],
|
||||
blocking: false,
|
||||
},
|
||||
},
|
||||
publicProbe: {
|
||||
ready: publicProbes.ready === true,
|
||||
@@ -1520,6 +1539,7 @@ export function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec,
|
||||
`--web-endpoint ${shellQuote(spec.publicWebUrl)}`,
|
||||
`--out ${shellQuote(renderDir)}`,
|
||||
].join(" "),
|
||||
`UNIDESK_RUNTIME_GITOPS_OVERLAY_B64="$overlay_b64" UNIDESK_RUNTIME_GITOPS_RUNTIME_PATH="$render_dir/${spec.runtimeRenderDir}" node ${shellQuote(rootPath("scripts/native/hwlab/runtime-gitops-postprocess.mjs"))}`,
|
||||
...nodeRuntimePipelinePostprocessScript(),
|
||||
].join("\n");
|
||||
return { result: runNodeHostScriptAsync(spec, script, timeoutSeconds, `${spec.nodeId.toLowerCase()}-${spec.lane}-render`), renderDir, worktreeDir, location: "node-host" };
|
||||
@@ -1946,6 +1966,27 @@ export function nodeRuntimePipelinePostprocessScript(): string[] {
|
||||
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED', String(kafka.enabled && kafka.features.transactionalProjector)) || codeAgentRuntimeChanged;",
|
||||
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED', String(kafka.enabled && kafka.features.projectionOutboxRelay)) || codeAgentRuntimeChanged;",
|
||||
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED', String(kafka.enabled && kafka.features.projectionRealtime)) || codeAgentRuntimeChanged;",
|
||||
" if (kafka.enabled && kafka.features.transactionalProjector) {",
|
||||
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS', String(kafka.transactionalProjector.heartbeatIntervalMs)) || codeAgentRuntimeChanged;",
|
||||
" } else {",
|
||||
" codeAgentRuntimeChanged = removeEnvValues(container, ['HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS']) || codeAgentRuntimeChanged;",
|
||||
" }",
|
||||
" if (kafka.enabled && kafka.features.projectionOutboxRelay) {",
|
||||
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS', String(kafka.projectionOutboxRelay.intervalMs)) || codeAgentRuntimeChanged;",
|
||||
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE', String(kafka.projectionOutboxRelay.batchSize)) || codeAgentRuntimeChanged;",
|
||||
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS', String(kafka.projectionOutboxRelay.leaseMs)) || codeAgentRuntimeChanged;",
|
||||
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS', String(kafka.projectionOutboxRelay.sendTimeoutMs)) || codeAgentRuntimeChanged;",
|
||||
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS', String(kafka.projectionOutboxRelay.retryBackoffMs)) || codeAgentRuntimeChanged;",
|
||||
" } else {",
|
||||
" codeAgentRuntimeChanged = removeEnvValues(container, ['HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS', 'HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE', 'HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS', 'HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS', 'HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS']) || codeAgentRuntimeChanged;",
|
||||
" }",
|
||||
" if (kafka.enabled && kafka.features.projectionRealtime) {",
|
||||
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE', String(kafka.projectionRealtime.outboxTailBatchSize)) || codeAgentRuntimeChanged;",
|
||||
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_WORKBENCH_SSE_HEARTBEAT_MS', String(kafka.projectionRealtime.sseHeartbeatMs)) || codeAgentRuntimeChanged;",
|
||||
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_WORKBENCH_SSE_DRAIN_TIMEOUT_MS', String(kafka.projectionRealtime.sseDrainTimeoutMs)) || codeAgentRuntimeChanged;",
|
||||
" } else {",
|
||||
" codeAgentRuntimeChanged = removeEnvValues(container, ['HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE', 'HWLAB_WORKBENCH_SSE_HEARTBEAT_MS', 'HWLAB_WORKBENCH_SSE_DRAIN_TIMEOUT_MS']) || codeAgentRuntimeChanged;",
|
||||
" }",
|
||||
" }",
|
||||
" }",
|
||||
" }",
|
||||
|
||||
@@ -7,7 +7,7 @@ import { test } from "bun:test";
|
||||
import { hwlabRuntimeLaneSpecForNode } from "../hwlab-node-lanes";
|
||||
import { buildWebObserveCommandVisibility, nodeWebProbeRealtimeFanoutProfiles, nodeWebProbeWorkbenchKafkaDebugReplay, nodeWebProbeWorkbenchTraceReadability, resolveWebObserveActionJson } from "./web-probe-observe-actions";
|
||||
|
||||
test("realtime fanout runner contract derives topics, groups, and independent capabilities from owning YAML", () => {
|
||||
test("realtime authority runner contract derives topics, groups, and transactional capabilities from owning YAML", () => {
|
||||
const profiles = nodeWebProbeRealtimeFanoutProfiles(hwlabRuntimeLaneSpecForNode("v03", "NC01"));
|
||||
const expectedKafka = profiles["pure-kafka-live"]?.expectedKafka as Record<string, any>;
|
||||
assert.deepEqual(expectedKafka.topics, {
|
||||
@@ -21,12 +21,34 @@ test("realtime fanout runner contract derives topics, groups, and independent ca
|
||||
liveSse: "hwlab-v03-workbench-live-sse",
|
||||
});
|
||||
assert.deepEqual(expectedKafka.capabilities, {
|
||||
directPublish: true,
|
||||
liveKafkaSse: true,
|
||||
kafkaRefreshReplay: true,
|
||||
transactionalProjector: false,
|
||||
projectionOutboxRelay: false,
|
||||
projectionRealtime: false,
|
||||
directPublish: false,
|
||||
liveKafkaSse: false,
|
||||
kafkaRefreshReplay: false,
|
||||
transactionalProjector: true,
|
||||
projectionOutboxRelay: true,
|
||||
projectionRealtime: true,
|
||||
});
|
||||
const kafka = hwlabRuntimeLaneSpecForNode("v03", "NC01").codeAgentRuntime?.kafkaEventBridge;
|
||||
assert.equal(kafka?.authority, "transactional-projection");
|
||||
assert.deepEqual(kafka?.transactionalProjector, { heartbeatIntervalMs: 2_000 });
|
||||
assert.deepEqual(kafka?.projectionOutboxRelay, {
|
||||
intervalMs: 250,
|
||||
batchSize: 100,
|
||||
leaseMs: 30_000,
|
||||
sendTimeoutMs: 10_000,
|
||||
retryBackoffMs: 1_000,
|
||||
});
|
||||
assert.deepEqual(kafka?.projectionRealtime, {
|
||||
outboxTailBatchSize: 100,
|
||||
sseHeartbeatMs: 15_000,
|
||||
sseDrainTimeoutMs: 2_500,
|
||||
});
|
||||
assert.deepEqual(kafka?.healthThresholds, {
|
||||
consumerLagWarning: 0,
|
||||
outboxBacklogWarning: 0,
|
||||
retryCountWarning: 0,
|
||||
failedInboxWarning: 0,
|
||||
dlqWarning: 0,
|
||||
});
|
||||
assert.deepEqual(profiles["pure-kafka-live"]?.requiredEventTypes, {
|
||||
agentrun: ["user_message", "assistant_message", "terminal_status"],
|
||||
|
||||
@@ -1482,6 +1482,7 @@ export function nodeRuntimeCodeAgentRuntimeStatus(spec: HwlabRuntimeLaneSpec, na
|
||||
const apiKey = secretKeyStatus(spec, runtime.apiKeySecretName, runtime.apiKeySecretKey);
|
||||
const cloudApiEnv = nodeRuntimeCodeAgentCloudApiEnvStatus(spec, runtime);
|
||||
const cloudWebEnv = nodeRuntimeCodeAgentCloudWebEnvStatus(spec, runtime);
|
||||
const kafkaAuthority = nodeRuntimeKafkaAuthorityStatus(spec, runtime);
|
||||
const ready = runnerNamespace.exitCode === 0
|
||||
&& secretNamespace.exitCode === 0
|
||||
&& managerNamespace.exitCode === 0
|
||||
@@ -1538,12 +1539,70 @@ export function nodeRuntimeCodeAgentRuntimeStatus(spec: HwlabRuntimeLaneSpec, na
|
||||
apiKey,
|
||||
cloudApiEnv,
|
||||
cloudWebEnv,
|
||||
kafkaAuthority,
|
||||
repoUrlFrom: runtime.repoUrlFrom,
|
||||
providerIdFrom: runtime.providerIdFrom,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function nodeRuntimeKafkaAuthorityStatus(spec: HwlabRuntimeLaneSpec, runtime: NonNullable<HwlabRuntimeLaneSpec["codeAgentRuntime"]>): Record<string, unknown> {
|
||||
const kafka = runtime.kafkaEventBridge;
|
||||
if (kafka === undefined || kafka.enabled === false) return { required: false, authority: kafka?.authority ?? null, valuesPrinted: false };
|
||||
const healthUrl = `${spec.publicApiUrl.replace(/\/+$/u, "")}/health/ready`;
|
||||
const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", healthUrl], repoRoot, { timeoutMs: 15_000 });
|
||||
let health: Record<string, unknown> = {};
|
||||
try {
|
||||
health = result.exitCode === 0 ? record(JSON.parse(result.stdout)) : {};
|
||||
} catch {
|
||||
health = {};
|
||||
}
|
||||
const projector = record(health.kafkaProjector);
|
||||
const components = Array.isArray(projector.components) ? projector.components.map(record) : [];
|
||||
const observations = [projector, ...components];
|
||||
const consumerLag = observations.map((item) => record(item.consumerLag).total).find((value) => typeof value === "number") ?? null;
|
||||
const backlogCount = observations.map((item) => item.backlogCount).find((value) => typeof value === "number") ?? null;
|
||||
const retryCount = observations.map((item) => item.retryCount).find((value) => typeof value === "number") ?? null;
|
||||
const failedInboxCount = observations.map((item) => item.failedInboxCount).find((value) => typeof value === "number") ?? null;
|
||||
const dlqCount = observations.map((item) => item.dlqCount).find((value) => typeof value === "number") ?? null;
|
||||
const thresholds = kafka.healthThresholds;
|
||||
const warningReasons = [
|
||||
typeof consumerLag === "number" && consumerLag > thresholds.consumerLagWarning ? "consumer-lag" : null,
|
||||
typeof backlogCount === "number" && backlogCount > thresholds.outboxBacklogWarning ? "outbox-backlog" : null,
|
||||
typeof retryCount === "number" && retryCount > thresholds.retryCountWarning ? "outbox-retry" : null,
|
||||
typeof failedInboxCount === "number" && failedInboxCount > thresholds.failedInboxWarning ? "failed-inbox" : null,
|
||||
typeof dlqCount === "number" && dlqCount > thresholds.dlqWarning ? "dlq" : null,
|
||||
].filter((value): value is string => value !== null);
|
||||
return {
|
||||
required: true,
|
||||
authority: kafka.authority,
|
||||
fixedGroups: {
|
||||
directPublish: kafka.directPublishConsumerGroupId,
|
||||
transactionalProjector: kafka.transactionalProjectorConsumerGroupId,
|
||||
liveSse: kafka.hwlabEventConsumerGroupId,
|
||||
},
|
||||
capabilities: kafka.features,
|
||||
liveReplay: {
|
||||
live: kafka.features.projectionRealtime || kafka.features.liveKafkaSse,
|
||||
replay: kafka.features.projectionRealtime || kafka.features.kafkaRefreshReplay,
|
||||
cursor: kafka.features.projectionRealtime ? "projection-outbox-seq" : kafka.features.kafkaRefreshReplay ? "kafka-refresh-replay" : "none",
|
||||
},
|
||||
healthObserved: Object.keys(projector).length > 0,
|
||||
healthStatus: projector.status ?? null,
|
||||
consumerLag,
|
||||
backlogCount,
|
||||
retryCount,
|
||||
failedInboxCount,
|
||||
dlqCount,
|
||||
thresholds,
|
||||
warning: warningReasons.length > 0,
|
||||
warningReasons,
|
||||
blocking: false,
|
||||
result: compactRuntimeCommand(result),
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function nodeRuntimeCodeAgentManagerTarget(managerUrl: string, fallbackNamespace: string): { namespace: string; serviceName: string } {
|
||||
let parsed: URL | null = null;
|
||||
try {
|
||||
@@ -1666,13 +1725,47 @@ function nodeRuntimeCodeAgentCloudApiEnvStatus(spec: HwlabRuntimeLaneSpec, runti
|
||||
expectValue("HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.transactionalProjector));
|
||||
expectValue("HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.projectionOutboxRelay));
|
||||
expectValue("HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.projectionRealtime));
|
||||
if (kafkaEventBridge.enabled && kafkaEventBridge.features.transactionalProjector) {
|
||||
if (kafkaEventBridge.transactionalProjector === undefined) throw new Error("codeAgentRuntime.kafkaEventBridge.transactionalProjector is required when transactional projector is enabled");
|
||||
expectValue("HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS", String(kafkaEventBridge.transactionalProjector.heartbeatIntervalMs));
|
||||
} else {
|
||||
expectAbsent("HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS");
|
||||
}
|
||||
const relayEnvNames = [
|
||||
"HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS",
|
||||
"HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE",
|
||||
"HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS",
|
||||
"HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS",
|
||||
"HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS",
|
||||
];
|
||||
if (kafkaEventBridge.enabled && kafkaEventBridge.features.projectionOutboxRelay) {
|
||||
const relay = kafkaEventBridge.projectionOutboxRelay;
|
||||
if (relay === undefined) throw new Error("codeAgentRuntime.kafkaEventBridge.projectionOutboxRelay is required when projection outbox relay is enabled");
|
||||
expectValue("HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS", String(relay.intervalMs));
|
||||
expectValue("HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE", String(relay.batchSize));
|
||||
expectValue("HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS", String(relay.leaseMs));
|
||||
expectValue("HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS", String(relay.sendTimeoutMs));
|
||||
expectValue("HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS", String(relay.retryBackoffMs));
|
||||
} else {
|
||||
for (const name of relayEnvNames) expectAbsent(name);
|
||||
}
|
||||
const projectionRealtimeEnvNames = ["HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE", "HWLAB_WORKBENCH_SSE_HEARTBEAT_MS", "HWLAB_WORKBENCH_SSE_DRAIN_TIMEOUT_MS"];
|
||||
if (kafkaEventBridge.enabled && kafkaEventBridge.features.projectionRealtime) {
|
||||
const realtime = kafkaEventBridge.projectionRealtime;
|
||||
if (realtime === undefined) throw new Error("codeAgentRuntime.kafkaEventBridge.projectionRealtime is required when projection realtime is enabled");
|
||||
expectValue("HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE", String(realtime.outboxTailBatchSize));
|
||||
expectValue("HWLAB_WORKBENCH_SSE_HEARTBEAT_MS", String(realtime.sseHeartbeatMs));
|
||||
expectValue("HWLAB_WORKBENCH_SSE_DRAIN_TIMEOUT_MS", String(realtime.sseDrainTimeoutMs));
|
||||
} else {
|
||||
for (const name of projectionRealtimeEnvNames) expectAbsent(name);
|
||||
}
|
||||
}
|
||||
return {
|
||||
ready: mismatches.length === 0,
|
||||
deploymentExists: true,
|
||||
deploymentName: "hwlab-cloud-api",
|
||||
containerName: "hwlab-cloud-api",
|
||||
checkedEnvCount: 9 + (kafkaShadowProducer === undefined ? 0 : 5) + (kafkaEventBridge === undefined ? 0 : 21),
|
||||
checkedEnvCount: 9 + (kafkaShadowProducer === undefined ? 0 : 5) + (kafkaEventBridge === undefined ? 0 : 30),
|
||||
envCount: envByName.size,
|
||||
mismatches,
|
||||
result: compactCodeAgentDeploymentProbeResult(envProbe),
|
||||
|
||||
Reference in New Issue
Block a user