feat: add code queue execution plane inspection
This commit is contained in:
Codex
@@ -0,0 +1,997 @@
|
||||
import { runCommand, type CommandResult } from "./command";
|
||||
import { repoRoot } from "./config";
|
||||
import { classifyD601K3sTarget, d601NativeKubeconfig, d601RequiredNodeName, type D601K3sGuardClassification } from "./d601-k3s-guard";
|
||||
|
||||
type PlaneStatus = "ready" | "blocked" | "degraded";
|
||||
type SignalSeverity = "blocker" | "warning";
|
||||
|
||||
interface ExecutionPlaneOptions {
|
||||
namespace: string;
|
||||
kubeconfig: string;
|
||||
worktreePath: string;
|
||||
full: boolean;
|
||||
raw: boolean;
|
||||
skipProbe: boolean;
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
interface ProbeResult {
|
||||
ok: boolean;
|
||||
command: string[];
|
||||
exitCode: number | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
timedOut: boolean;
|
||||
}
|
||||
|
||||
interface DeploymentObservation {
|
||||
name: string;
|
||||
namespace: string;
|
||||
observed: boolean;
|
||||
expectedRole: "scheduler" | "read" | "write";
|
||||
expectedSchedulerEnabled: string;
|
||||
labels: {
|
||||
app: string | null;
|
||||
component: string | null;
|
||||
deploymentMode: string | null;
|
||||
instanceId: string | null;
|
||||
};
|
||||
annotations: {
|
||||
deployRef: string | null;
|
||||
deployCommit: string | null;
|
||||
deployRequestedCommit: string | null;
|
||||
imageSource: string | null;
|
||||
};
|
||||
replicas: {
|
||||
desired: number | null;
|
||||
ready: number;
|
||||
available: number;
|
||||
updated: number;
|
||||
};
|
||||
nodeSelector: string | null;
|
||||
image: string | null;
|
||||
env: {
|
||||
serviceRole: string | null;
|
||||
schedulerEnabled: string | null;
|
||||
unideskDeployCommit: string | null;
|
||||
unideskDeployRequestedCommit: string | null;
|
||||
codeQueueDeployCommit: string | null;
|
||||
codeQueueDeployRequestedCommit: string | null;
|
||||
};
|
||||
repoHostPath: string | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
interface PodObservation {
|
||||
name: string;
|
||||
instanceId: string | null;
|
||||
component: string | null;
|
||||
nodeName: string | null;
|
||||
phase: string | null;
|
||||
ready: boolean;
|
||||
imageID: string | null;
|
||||
digest: string | null;
|
||||
}
|
||||
|
||||
interface ServiceObservation {
|
||||
name: string;
|
||||
observed: boolean;
|
||||
type: string | null;
|
||||
clusterIP: string | null;
|
||||
ports: string[];
|
||||
selector: {
|
||||
app: string | null;
|
||||
component: string | null;
|
||||
instanceId: string | null;
|
||||
};
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
interface WorktreeObservation {
|
||||
path: string;
|
||||
ok: boolean;
|
||||
head: string | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
interface ResidualObservation {
|
||||
composeBackend: {
|
||||
ok: boolean;
|
||||
present: boolean;
|
||||
containers: Array<{ name: string; status: string | null; image: string | null }>;
|
||||
error: string | null;
|
||||
};
|
||||
loopbackPort4222: {
|
||||
ok: boolean;
|
||||
present: boolean;
|
||||
listeners: Array<{ localAddress: string; process: string | null; line: string }>;
|
||||
error: string | null;
|
||||
};
|
||||
}
|
||||
|
||||
interface JudgeProbeObservation {
|
||||
ok: boolean;
|
||||
attempted: boolean;
|
||||
behaviorVersion: string | null;
|
||||
expectedBehaviorVersion: string;
|
||||
configured: boolean | null;
|
||||
model: string | null;
|
||||
hits: number | null;
|
||||
total: number | null;
|
||||
hitRate: number | null;
|
||||
serviceProxyPath: string | null;
|
||||
error: string | null;
|
||||
raw: unknown;
|
||||
}
|
||||
|
||||
export interface CodeQueueExecutionPlaneObservation {
|
||||
checkedAt: string;
|
||||
namespace: string;
|
||||
kubeconfig: string;
|
||||
worktreePath: string;
|
||||
guard: D601K3sGuardClassification;
|
||||
deployments: DeploymentObservation[];
|
||||
pods: PodObservation[];
|
||||
services: ServiceObservation[];
|
||||
worktree: WorktreeObservation;
|
||||
residual: ResidualObservation;
|
||||
judgeProbe: JudgeProbeObservation;
|
||||
commandDiagnostics: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface ExecutionPlaneCollector {
|
||||
collect(options: ExecutionPlaneOptions): Promise<CodeQueueExecutionPlaneObservation> | CodeQueueExecutionPlaneObservation;
|
||||
}
|
||||
|
||||
interface DriftSignal {
|
||||
code: string;
|
||||
severity: SignalSeverity;
|
||||
message: string;
|
||||
expected?: unknown;
|
||||
observed?: unknown;
|
||||
field?: string;
|
||||
}
|
||||
|
||||
const expectedNamespace = "unidesk";
|
||||
const expectedWorktreePath = "/home/ubuntu/cq-deploy";
|
||||
export const expectedJudgeProbeBehaviorVersion = "code-queue-judge-probe:v1";
|
||||
|
||||
const expectedDeployments = [
|
||||
{ name: "code-queue", role: "scheduler" as const, schedulerEnabled: "true" },
|
||||
{ name: "code-queue-read", role: "read" as const, schedulerEnabled: "false" },
|
||||
{ name: "code-queue-write", role: "write" as const, schedulerEnabled: "false" },
|
||||
];
|
||||
|
||||
function nowIso(): string {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
function hasFlag(args: string[], name: string): boolean {
|
||||
return args.includes(name);
|
||||
}
|
||||
|
||||
function optionValue(args: string[], names: string[]): string | undefined {
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
if (!names.includes(args[index] ?? "")) continue;
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error(`${args[index]} requires a value`);
|
||||
return value;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value: string | undefined, fallback: number, max: number): number {
|
||||
if (value === undefined) return fallback;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) throw new Error("--timeout-ms must be a positive integer");
|
||||
return Math.min(parsed, max);
|
||||
}
|
||||
|
||||
function parseExecutionPlaneOptions(args: string[]): ExecutionPlaneOptions {
|
||||
const knownFlags = new Set(["--full", "--raw", "--skip-probe"]);
|
||||
const knownValues = new Set(["--namespace", "--kubeconfig", "--worktree", "--timeout-ms"]);
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index] ?? "";
|
||||
if (!arg.startsWith("--")) throw new Error(`codex execution-plane does not accept positional argument: ${arg}`);
|
||||
if (knownFlags.has(arg)) continue;
|
||||
if (knownValues.has(arg)) {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`unknown codex execution-plane option: ${arg}`);
|
||||
}
|
||||
const raw = hasFlag(args, "--raw");
|
||||
return {
|
||||
namespace: optionValue(args, ["--namespace"]) ?? expectedNamespace,
|
||||
kubeconfig: optionValue(args, ["--kubeconfig"]) ?? d601NativeKubeconfig,
|
||||
worktreePath: optionValue(args, ["--worktree"]) ?? expectedWorktreePath,
|
||||
full: raw || hasFlag(args, "--full"),
|
||||
raw,
|
||||
skipProbe: hasFlag(args, "--skip-probe"),
|
||||
timeoutMs: parsePositiveInteger(optionValue(args, ["--timeout-ms"]), 15_000, 60_000),
|
||||
};
|
||||
}
|
||||
|
||||
function firstLine(text: string): string | null {
|
||||
const line = text.split(/\r?\n/u).map((item) => item.trim()).find((item) => item.length > 0);
|
||||
return line ?? null;
|
||||
}
|
||||
|
||||
function lines(text: string): string[] {
|
||||
return text.split(/\r?\n/u).map((item) => item.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
function nullable(value: string | undefined): string | null {
|
||||
const trimmed = (value ?? "").trim();
|
||||
return trimmed.length === 0 ? null : trimmed;
|
||||
}
|
||||
|
||||
function numberOrNull(value: string | undefined): number | null {
|
||||
const trimmed = (value ?? "").trim();
|
||||
if (trimmed.length === 0) return null;
|
||||
const parsed = Number(trimmed);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function boolFromString(value: string | null): boolean {
|
||||
return value === "true";
|
||||
}
|
||||
|
||||
function commandProbe(result: CommandResult): ProbeResult {
|
||||
return {
|
||||
ok: result.exitCode === 0 && !result.timedOut,
|
||||
command: result.command,
|
||||
exitCode: result.exitCode,
|
||||
stdout: result.stdout,
|
||||
stderr: result.stderr,
|
||||
timedOut: result.timedOut,
|
||||
};
|
||||
}
|
||||
|
||||
function safeError(probe: ProbeResult): string | null {
|
||||
if (probe.ok) return null;
|
||||
const text = firstLine(probe.stderr) ?? firstLine(probe.stdout);
|
||||
return text ?? `exitCode=${probe.exitCode ?? "null"}`;
|
||||
}
|
||||
|
||||
function runKubectl(args: string[], options: ExecutionPlaneOptions): ProbeResult {
|
||||
return commandProbe(runCommand(["kubectl", ...args], repoRoot, {
|
||||
timeoutMs: options.timeoutMs,
|
||||
env: { ...process.env, KUBECONFIG: options.kubeconfig },
|
||||
}));
|
||||
}
|
||||
|
||||
function collectGuard(options: ExecutionPlaneOptions): { guard: D601K3sGuardClassification; diagnostics: Record<string, unknown> } {
|
||||
const context = runKubectl(["config", "current-context"], options);
|
||||
const server = runKubectl(["config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], options);
|
||||
const nodes = runKubectl(["get", "nodes", "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}"], options);
|
||||
const combinedText = [context.stdout, context.stderr, server.stdout, server.stderr, nodes.stdout, nodes.stderr].join("\n");
|
||||
const guard = classifyD601K3sTarget({
|
||||
kubeconfig: options.kubeconfig,
|
||||
expectedKubeconfig: d601NativeKubeconfig,
|
||||
currentContext: firstLine(context.stdout),
|
||||
apiServer: firstLine(server.stdout),
|
||||
nodeNames: lines(nodes.stdout),
|
||||
commandsOk: context.ok && server.ok && nodes.ok,
|
||||
combinedText,
|
||||
});
|
||||
return {
|
||||
guard,
|
||||
diagnostics: {
|
||||
context: compactProbe(context),
|
||||
server: compactProbe(server),
|
||||
nodes: compactProbe(nodes),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function compactProbe(probe: ProbeResult): Record<string, unknown> {
|
||||
return {
|
||||
ok: probe.ok,
|
||||
exitCode: probe.exitCode,
|
||||
timedOut: probe.timedOut,
|
||||
stdoutPreview: firstLine(probe.stdout),
|
||||
stderrPreview: firstLine(probe.stderr),
|
||||
};
|
||||
}
|
||||
|
||||
function deploymentJsonPath(): string {
|
||||
return [
|
||||
"{.metadata.name}",
|
||||
"{\"\\t\"}{.metadata.namespace}",
|
||||
"{\"\\t\"}{.metadata.labels.app\\.kubernetes\\.io/name}",
|
||||
"{\"\\t\"}{.metadata.labels.app\\.kubernetes\\.io/component}",
|
||||
"{\"\\t\"}{.metadata.labels.unidesk\\.ai/deployment-mode}",
|
||||
"{\"\\t\"}{.metadata.labels.unidesk\\.ai/instance-id}",
|
||||
"{\"\\t\"}{.metadata.annotations.unidesk\\.ai/deploy-ref}",
|
||||
"{\"\\t\"}{.metadata.annotations.unidesk\\.ai/deploy-commit}",
|
||||
"{\"\\t\"}{.metadata.annotations.unidesk\\.ai/deploy-requested-commit}",
|
||||
"{\"\\t\"}{.metadata.annotations.unidesk\\.ai/image-source}",
|
||||
"{\"\\t\"}{.status.readyReplicas}",
|
||||
"{\"\\t\"}{.status.availableReplicas}",
|
||||
"{\"\\t\"}{.status.updatedReplicas}",
|
||||
"{\"\\t\"}{.spec.replicas}",
|
||||
"{\"\\t\"}{.spec.template.spec.nodeSelector.unidesk\\.ai/node-id}",
|
||||
"{\"\\t\"}{.spec.template.spec.containers[?(@.name==\"code-queue\")].image}",
|
||||
"{\"\\t\"}{.spec.template.spec.containers[?(@.name==\"code-queue\")].env[?(@.name==\"CODE_QUEUE_SERVICE_ROLE\")].value}",
|
||||
"{\"\\t\"}{.spec.template.spec.containers[?(@.name==\"code-queue\")].env[?(@.name==\"CODE_QUEUE_SCHEDULER_ENABLED\")].value}",
|
||||
"{\"\\t\"}{.spec.template.spec.containers[?(@.name==\"code-queue\")].env[?(@.name==\"UNIDESK_DEPLOY_COMMIT\")].value}",
|
||||
"{\"\\t\"}{.spec.template.spec.containers[?(@.name==\"code-queue\")].env[?(@.name==\"UNIDESK_DEPLOY_REQUESTED_COMMIT\")].value}",
|
||||
"{\"\\t\"}{.spec.template.spec.containers[?(@.name==\"code-queue\")].env[?(@.name==\"CODE_QUEUE_DEPLOY_COMMIT\")].value}",
|
||||
"{\"\\t\"}{.spec.template.spec.containers[?(@.name==\"code-queue\")].env[?(@.name==\"CODE_QUEUE_DEPLOY_REQUESTED_COMMIT\")].value}",
|
||||
"{\"\\t\"}{.spec.template.spec.volumes[?(@.name==\"repo\")].hostPath.path}",
|
||||
].join("");
|
||||
}
|
||||
|
||||
function parseDeployment(fields: string[], expected: typeof expectedDeployments[number], namespace: string, error: string | null): DeploymentObservation {
|
||||
return {
|
||||
name: nullable(fields[0]) ?? expected.name,
|
||||
namespace: nullable(fields[1]) ?? namespace,
|
||||
observed: error === null && nullable(fields[0]) !== null,
|
||||
expectedRole: expected.role,
|
||||
expectedSchedulerEnabled: expected.schedulerEnabled,
|
||||
labels: {
|
||||
app: nullable(fields[2]),
|
||||
component: nullable(fields[3]),
|
||||
deploymentMode: nullable(fields[4]),
|
||||
instanceId: nullable(fields[5]),
|
||||
},
|
||||
annotations: {
|
||||
deployRef: nullable(fields[6]),
|
||||
deployCommit: nullable(fields[7]),
|
||||
deployRequestedCommit: nullable(fields[8]),
|
||||
imageSource: nullable(fields[9]),
|
||||
},
|
||||
replicas: {
|
||||
ready: numberOrNull(fields[10]) ?? 0,
|
||||
available: numberOrNull(fields[11]) ?? 0,
|
||||
updated: numberOrNull(fields[12]) ?? 0,
|
||||
desired: numberOrNull(fields[13]),
|
||||
},
|
||||
nodeSelector: nullable(fields[14]),
|
||||
image: nullable(fields[15]),
|
||||
env: {
|
||||
serviceRole: nullable(fields[16]),
|
||||
schedulerEnabled: nullable(fields[17]),
|
||||
unideskDeployCommit: nullable(fields[18]),
|
||||
unideskDeployRequestedCommit: nullable(fields[19]),
|
||||
codeQueueDeployCommit: nullable(fields[20]),
|
||||
codeQueueDeployRequestedCommit: nullable(fields[21]),
|
||||
},
|
||||
repoHostPath: nullable(fields[22]),
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
||||
function collectDeployments(options: ExecutionPlaneOptions): DeploymentObservation[] {
|
||||
return expectedDeployments.map((expected) => {
|
||||
const probe = runKubectl(["-n", options.namespace, "get", "deployment", expected.name, "-o", `jsonpath=${deploymentJsonPath()}`], options);
|
||||
const error = safeError(probe);
|
||||
const fields = probe.ok ? probe.stdout.trimEnd().split("\t") : [];
|
||||
return parseDeployment(fields, expected, options.namespace, error);
|
||||
});
|
||||
}
|
||||
|
||||
function collectPods(options: ExecutionPlaneOptions): PodObservation[] {
|
||||
const path = [
|
||||
"{range .items[*]}",
|
||||
"{.metadata.name}{\"\\t\"}",
|
||||
"{.metadata.labels.unidesk\\.ai/instance-id}{\"\\t\"}",
|
||||
"{.metadata.labels.app\\.kubernetes\\.io/component}{\"\\t\"}",
|
||||
"{.spec.nodeName}{\"\\t\"}",
|
||||
"{.status.phase}{\"\\t\"}",
|
||||
"{.status.containerStatuses[?(@.name==\"code-queue\")].ready}{\"\\t\"}",
|
||||
"{.status.containerStatuses[?(@.name==\"code-queue\")].imageID}{\"\\n\"}",
|
||||
"{end}",
|
||||
].join("");
|
||||
const probe = runKubectl(["-n", options.namespace, "get", "pods", "-l", "app.kubernetes.io/name=code-queue", "-o", `jsonpath=${path}`], options);
|
||||
if (!probe.ok) return [];
|
||||
return lines(probe.stdout).map((line) => {
|
||||
const fields = line.split("\t");
|
||||
const imageID = nullable(fields[6]);
|
||||
return {
|
||||
name: nullable(fields[0]) ?? "unknown",
|
||||
instanceId: nullable(fields[1]),
|
||||
component: nullable(fields[2]),
|
||||
nodeName: nullable(fields[3]),
|
||||
phase: nullable(fields[4]),
|
||||
ready: boolFromString(nullable(fields[5])),
|
||||
imageID,
|
||||
digest: digestFromText(imageID),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function serviceJsonPath(): string {
|
||||
return [
|
||||
"{.metadata.name}",
|
||||
"{\"\\t\"}{.spec.type}",
|
||||
"{\"\\t\"}{.spec.clusterIP}",
|
||||
"{\"\\t\"}{range .spec.ports[*]}{.name}:{.port}->{.targetPort}{\",\"}{end}",
|
||||
"{\"\\t\"}{.spec.selector.app\\.kubernetes\\.io/name}",
|
||||
"{\"\\t\"}{.spec.selector.app\\.kubernetes\\.io/component}",
|
||||
"{\"\\t\"}{.spec.selector.unidesk\\.ai/instance-id}",
|
||||
].join("");
|
||||
}
|
||||
|
||||
function collectServices(options: ExecutionPlaneOptions): ServiceObservation[] {
|
||||
return expectedDeployments.map((expected) => {
|
||||
const probe = runKubectl(["-n", options.namespace, "get", "service", expected.name, "-o", `jsonpath=${serviceJsonPath()}`], options);
|
||||
const error = safeError(probe);
|
||||
const fields = probe.ok ? probe.stdout.trimEnd().split("\t") : [];
|
||||
return {
|
||||
name: nullable(fields[0]) ?? expected.name,
|
||||
observed: error === null && nullable(fields[0]) !== null,
|
||||
type: nullable(fields[1]),
|
||||
clusterIP: nullable(fields[2]),
|
||||
ports: (nullable(fields[3]) ?? "").split(",").map((item) => item.trim()).filter(Boolean),
|
||||
selector: {
|
||||
app: nullable(fields[4]),
|
||||
component: nullable(fields[5]),
|
||||
instanceId: nullable(fields[6]),
|
||||
},
|
||||
error,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function collectWorktree(options: ExecutionPlaneOptions): WorktreeObservation {
|
||||
const probe = commandProbe(runCommand(["git", "-C", options.worktreePath, "rev-parse", "HEAD"], repoRoot, { timeoutMs: 5_000 }));
|
||||
return {
|
||||
path: options.worktreePath,
|
||||
ok: probe.ok,
|
||||
head: probe.ok ? firstLine(probe.stdout) : null,
|
||||
error: safeError(probe),
|
||||
};
|
||||
}
|
||||
|
||||
function collectResidual(): ResidualObservation {
|
||||
const docker = commandProbe(runCommand(["docker", "ps", "-a", "--filter", "name=code-queue-backend", "--format", "{{.Names}}\t{{.Status}}\t{{.Image}}"], repoRoot, { timeoutMs: 8_000 }));
|
||||
const containers = docker.ok
|
||||
? lines(docker.stdout).map((line) => {
|
||||
const fields = line.split("\t");
|
||||
return { name: fields[0] ?? "unknown", status: nullable(fields[1]), image: nullable(fields[2]) };
|
||||
}).filter((item) => item.name === "code-queue-backend" || item.name.includes("code-queue-backend"))
|
||||
: [];
|
||||
|
||||
const ss = commandProbe(runCommand(["ss", "-H", "-ltnp"], repoRoot, { timeoutMs: 8_000 }));
|
||||
const listeners = ss.ok
|
||||
? (lines(ss.stdout)
|
||||
.filter((line) => line.includes(":4222"))
|
||||
.map((line) => {
|
||||
const fields = line.split(/\s+/u);
|
||||
const localAddress = fields[3] ?? fields[2] ?? "";
|
||||
const processText = fields.find((field) => field.includes("users:")) ?? null;
|
||||
return { localAddress, process: processText, line };
|
||||
})
|
||||
.filter((item) => item.localAddress.startsWith("127.0.0.1:4222") || item.localAddress.startsWith("[::1]:4222")))
|
||||
: [];
|
||||
|
||||
return {
|
||||
composeBackend: {
|
||||
ok: docker.ok,
|
||||
present: containers.length > 0,
|
||||
containers,
|
||||
error: safeError(docker),
|
||||
},
|
||||
loopbackPort4222: {
|
||||
ok: ss.ok,
|
||||
present: listeners.length > 0,
|
||||
listeners,
|
||||
error: safeError(ss),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function parseJson(text: string): unknown {
|
||||
try {
|
||||
return JSON.parse(text) as unknown;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
||||
}
|
||||
|
||||
function stringValue(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
function numberValue(value: unknown): number | null {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
function collectJudgeProbe(options: ExecutionPlaneOptions): JudgeProbeObservation {
|
||||
if (options.skipProbe) {
|
||||
return {
|
||||
ok: false,
|
||||
attempted: false,
|
||||
behaviorVersion: null,
|
||||
expectedBehaviorVersion: expectedJudgeProbeBehaviorVersion,
|
||||
configured: null,
|
||||
model: null,
|
||||
hits: null,
|
||||
total: null,
|
||||
hitRate: null,
|
||||
serviceProxyPath: null,
|
||||
error: "--skip-probe requested",
|
||||
raw: null,
|
||||
};
|
||||
}
|
||||
const paths = [
|
||||
`/api/v1/namespaces/${options.namespace}/services/code-queue/proxy/api/judge/probe`,
|
||||
`/api/v1/namespaces/${options.namespace}/services/code-queue:4222/proxy/api/judge/probe`,
|
||||
`/api/v1/namespaces/${options.namespace}/services/code-queue:http/proxy/api/judge/probe`,
|
||||
];
|
||||
for (const path of paths) {
|
||||
const probe = runKubectl(["--request-timeout=15s", "get", "--raw", path], options);
|
||||
if (!probe.ok) continue;
|
||||
const raw = parseJson(probe.stdout);
|
||||
const record = asRecord(raw);
|
||||
if (record === null) {
|
||||
return {
|
||||
ok: false,
|
||||
attempted: true,
|
||||
behaviorVersion: null,
|
||||
expectedBehaviorVersion: expectedJudgeProbeBehaviorVersion,
|
||||
configured: null,
|
||||
model: null,
|
||||
hits: null,
|
||||
total: null,
|
||||
hitRate: null,
|
||||
serviceProxyPath: path,
|
||||
error: "judge probe returned non-JSON response",
|
||||
raw: probe.stdout.slice(0, 2000),
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: record.ok === true,
|
||||
attempted: true,
|
||||
behaviorVersion: stringValue(record.behaviorVersion ?? asRecord(record.behavior)?.version ?? record.version),
|
||||
expectedBehaviorVersion: expectedJudgeProbeBehaviorVersion,
|
||||
configured: typeof record.configured === "boolean" ? record.configured : null,
|
||||
model: stringValue(record.model),
|
||||
hits: numberValue(record.hits),
|
||||
total: numberValue(record.total),
|
||||
hitRate: numberValue(record.hitRate),
|
||||
serviceProxyPath: path,
|
||||
error: record.ok === true ? null : stringValue(record.error) ?? "judge probe returned ok=false",
|
||||
raw,
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
attempted: true,
|
||||
behaviorVersion: null,
|
||||
expectedBehaviorVersion: expectedJudgeProbeBehaviorVersion,
|
||||
configured: null,
|
||||
model: null,
|
||||
hits: null,
|
||||
total: null,
|
||||
hitRate: null,
|
||||
serviceProxyPath: null,
|
||||
error: "judge probe could not be read through Kubernetes API service proxy",
|
||||
raw: null,
|
||||
};
|
||||
}
|
||||
|
||||
function digestFromText(value: string | null): string | null {
|
||||
if (value === null) return null;
|
||||
const match = value.match(/sha256:[0-9a-f]{64}/iu);
|
||||
return match?.[0].toLowerCase() ?? null;
|
||||
}
|
||||
|
||||
class LiveExecutionPlaneCollector implements ExecutionPlaneCollector {
|
||||
collect(options: ExecutionPlaneOptions): CodeQueueExecutionPlaneObservation {
|
||||
const { guard, diagnostics } = collectGuard(options);
|
||||
const residual = collectResidual();
|
||||
const worktree = collectWorktree(options);
|
||||
if (guard.status !== "pass") {
|
||||
return {
|
||||
checkedAt: nowIso(),
|
||||
namespace: options.namespace,
|
||||
kubeconfig: options.kubeconfig,
|
||||
worktreePath: options.worktreePath,
|
||||
guard,
|
||||
deployments: [],
|
||||
pods: [],
|
||||
services: [],
|
||||
worktree,
|
||||
residual,
|
||||
judgeProbe: {
|
||||
ok: false,
|
||||
attempted: false,
|
||||
behaviorVersion: null,
|
||||
expectedBehaviorVersion: expectedJudgeProbeBehaviorVersion,
|
||||
configured: null,
|
||||
model: null,
|
||||
hits: null,
|
||||
total: null,
|
||||
hitRate: null,
|
||||
serviceProxyPath: null,
|
||||
error: "D601 k3s guard did not pass; runtime probe skipped",
|
||||
raw: null,
|
||||
},
|
||||
commandDiagnostics: { guard: diagnostics },
|
||||
};
|
||||
}
|
||||
return {
|
||||
checkedAt: nowIso(),
|
||||
namespace: options.namespace,
|
||||
kubeconfig: options.kubeconfig,
|
||||
worktreePath: options.worktreePath,
|
||||
guard,
|
||||
deployments: collectDeployments(options),
|
||||
pods: collectPods(options),
|
||||
services: collectServices(options),
|
||||
worktree,
|
||||
residual,
|
||||
judgeProbe: collectJudgeProbe(options),
|
||||
commandDiagnostics: { guard: diagnostics },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function safeCommit(value: string | null): string | null {
|
||||
if (value === null) return null;
|
||||
const trimmed = value.trim().toLowerCase();
|
||||
if (!/^[0-9a-f]{7,40}$/u.test(trimmed)) return null;
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function commitMatches(left: string, right: string): boolean {
|
||||
const a = left.toLowerCase();
|
||||
const b = right.toLowerCase();
|
||||
return a === b || (a.length >= 7 && b.startsWith(a)) || (b.length >= 7 && a.startsWith(b));
|
||||
}
|
||||
|
||||
function uniqueStrings(values: Array<string | null>): string[] {
|
||||
return [...new Set(values.filter((value): value is string => value !== null && value.length > 0))];
|
||||
}
|
||||
|
||||
function deploymentCommitMarkers(deployment: DeploymentObservation): Record<string, string | null> {
|
||||
return {
|
||||
annotationDeployCommit: safeCommit(deployment.annotations.deployCommit),
|
||||
annotationDeployRequestedCommit: safeCommit(deployment.annotations.deployRequestedCommit),
|
||||
unideskDeployCommit: safeCommit(deployment.env.unideskDeployCommit),
|
||||
unideskDeployRequestedCommit: safeCommit(deployment.env.unideskDeployRequestedCommit),
|
||||
codeQueueDeployCommit: safeCommit(deployment.env.codeQueueDeployCommit),
|
||||
codeQueueDeployRequestedCommit: safeCommit(deployment.env.codeQueueDeployRequestedCommit),
|
||||
};
|
||||
}
|
||||
|
||||
function deploymentFormalSignals(deployment: DeploymentObservation): DriftSignal[] {
|
||||
const signals: DriftSignal[] = [];
|
||||
if (!deployment.observed) {
|
||||
signals.push({ code: "execution-plane-deployment-missing", severity: "blocker", message: `${deployment.name} deployment is missing`, expected: deployment.name, observed: deployment.error });
|
||||
return signals;
|
||||
}
|
||||
if (deployment.namespace !== expectedNamespace) {
|
||||
signals.push({ code: "execution-plane-namespace-mismatch", severity: "blocker", message: `${deployment.name} is not in ${expectedNamespace}`, expected: expectedNamespace, observed: deployment.namespace });
|
||||
}
|
||||
if (deployment.labels.app !== "code-queue") {
|
||||
signals.push({ code: "execution-plane-app-label-mismatch", severity: "blocker", message: `${deployment.name} app label is not code-queue`, expected: "code-queue", observed: deployment.labels.app });
|
||||
}
|
||||
if (deployment.labels.deploymentMode !== "k3sctl-managed") {
|
||||
signals.push({ code: "execution-plane-mode-mismatch", severity: "blocker", message: `${deployment.name} is not k3sctl-managed`, expected: "k3sctl-managed", observed: deployment.labels.deploymentMode });
|
||||
}
|
||||
if (deployment.expectedRole !== "scheduler" && deployment.labels.component !== deployment.expectedRole) {
|
||||
signals.push({ code: "execution-plane-component-mismatch", severity: "blocker", message: `${deployment.name} component label does not match role`, expected: deployment.expectedRole, observed: deployment.labels.component });
|
||||
}
|
||||
if (deployment.nodeSelector !== "D601" && deployment.nodeSelector !== "d601") {
|
||||
signals.push({ code: "execution-plane-node-selector-mismatch", severity: "blocker", message: `${deployment.name} is not pinned to D601`, expected: "D601", observed: deployment.nodeSelector });
|
||||
}
|
||||
if (deployment.env.serviceRole !== deployment.expectedRole) {
|
||||
signals.push({ code: "execution-plane-role-mismatch", severity: "blocker", message: `${deployment.name} CODE_QUEUE_SERVICE_ROLE mismatch`, expected: deployment.expectedRole, observed: deployment.env.serviceRole });
|
||||
}
|
||||
if (deployment.env.schedulerEnabled !== deployment.expectedSchedulerEnabled) {
|
||||
signals.push({ code: "execution-plane-scheduler-flag-mismatch", severity: "blocker", message: `${deployment.name} CODE_QUEUE_SCHEDULER_ENABLED mismatch`, expected: deployment.expectedSchedulerEnabled, observed: deployment.env.schedulerEnabled });
|
||||
}
|
||||
if (deployment.image === null || !deployment.image.includes("unidesk-code-queue")) {
|
||||
signals.push({ code: "execution-plane-image-mismatch", severity: "blocker", message: `${deployment.name} image is not a Code Queue image`, expected: "unidesk-code-queue", observed: deployment.image });
|
||||
}
|
||||
if (deployment.replicas.ready <= 0) {
|
||||
signals.push({ code: "execution-plane-no-ready-replica", severity: "blocker", message: `${deployment.name} has no ready replicas`, expected: "readyReplicas > 0", observed: deployment.replicas.ready });
|
||||
}
|
||||
return signals;
|
||||
}
|
||||
|
||||
function driftSignals(observation: CodeQueueExecutionPlaneObservation): DriftSignal[] {
|
||||
const signals: DriftSignal[] = [];
|
||||
const commitValues = observation.deployments.flatMap((deployment) => Object.values(deploymentCommitMarkers(deployment)));
|
||||
const uniqueCommits = uniqueStrings(commitValues);
|
||||
const worktreeHead = safeCommit(observation.worktree.head);
|
||||
|
||||
if (uniqueCommits.length === 0) {
|
||||
signals.push({
|
||||
code: "deployment-drift",
|
||||
severity: "blocker",
|
||||
field: "deployment.commitMarkers",
|
||||
message: "Code Queue deployments do not expose comparable deploy commit markers",
|
||||
expected: "UNIDESK_DEPLOY_COMMIT or CODE_QUEUE_DEPLOY_COMMIT",
|
||||
observed: "missing",
|
||||
});
|
||||
}
|
||||
if (uniqueCommits.length > 1) {
|
||||
signals.push({
|
||||
code: "deployment-drift",
|
||||
severity: "blocker",
|
||||
field: "deployment.commitMarkers",
|
||||
message: "Code Queue deployment commit markers disagree",
|
||||
observed: uniqueCommits,
|
||||
});
|
||||
}
|
||||
for (const deployment of observation.deployments) {
|
||||
if (uniqueStrings(Object.values(deploymentCommitMarkers(deployment))).length === 0) {
|
||||
signals.push({
|
||||
code: "deployment-drift",
|
||||
severity: "blocker",
|
||||
field: `deployment.${deployment.name}.commitMarkers`,
|
||||
message: `${deployment.name} does not expose comparable deploy commit markers`,
|
||||
expected: "UNIDESK_DEPLOY_COMMIT or CODE_QUEUE_DEPLOY_COMMIT",
|
||||
observed: "missing",
|
||||
});
|
||||
}
|
||||
}
|
||||
if (worktreeHead === null) {
|
||||
signals.push({
|
||||
code: "deployment-drift",
|
||||
severity: "blocker",
|
||||
field: "mountedWorktree.head",
|
||||
message: "Mounted Code Queue worktree HEAD could not be observed",
|
||||
expected: observation.worktreePath,
|
||||
observed: observation.worktree.error,
|
||||
});
|
||||
} else if (uniqueCommits.length > 0 && !uniqueCommits.some((commit) => commitMatches(commit, worktreeHead))) {
|
||||
signals.push({
|
||||
code: "deployment-drift",
|
||||
severity: "blocker",
|
||||
field: "mountedWorktree.head",
|
||||
message: "Mounted worktree HEAD does not match deployment commit markers",
|
||||
expected: uniqueCommits,
|
||||
observed: worktreeHead,
|
||||
});
|
||||
}
|
||||
|
||||
const repoHostPaths = uniqueStrings(observation.deployments.map((deployment) => deployment.repoHostPath));
|
||||
if (repoHostPaths.length !== 1 || repoHostPaths[0] !== observation.worktreePath) {
|
||||
signals.push({
|
||||
code: "deployment-drift",
|
||||
severity: "blocker",
|
||||
field: "deployment.repoHostPath",
|
||||
message: "Code Queue deployments do not all mount only the expected worktree path",
|
||||
expected: observation.worktreePath,
|
||||
observed: repoHostPaths,
|
||||
});
|
||||
}
|
||||
|
||||
const podDigests = uniqueStrings(observation.pods.map((pod) => pod.digest));
|
||||
if (podDigests.length === 0) {
|
||||
signals.push({
|
||||
code: "deployment-drift",
|
||||
severity: "blocker",
|
||||
field: "artifact.digest",
|
||||
message: "No running Code Queue pod image digest was observed",
|
||||
expected: "sha256 digest from pod status imageID",
|
||||
observed: "missing",
|
||||
});
|
||||
} else if (podDigests.length > 1) {
|
||||
signals.push({
|
||||
code: "deployment-drift",
|
||||
severity: "blocker",
|
||||
field: "artifact.digest",
|
||||
message: "Code Queue pod artifact digests disagree",
|
||||
observed: podDigests,
|
||||
});
|
||||
}
|
||||
|
||||
if (!observation.judgeProbe.attempted) {
|
||||
signals.push({
|
||||
code: "deployment-drift",
|
||||
severity: "warning",
|
||||
field: "judgeProbe.behaviorVersion",
|
||||
message: "/api/judge/probe behavior version was not checked",
|
||||
expected: expectedJudgeProbeBehaviorVersion,
|
||||
observed: observation.judgeProbe.error,
|
||||
});
|
||||
} else if (observation.judgeProbe.behaviorVersion !== expectedJudgeProbeBehaviorVersion) {
|
||||
signals.push({
|
||||
code: "deployment-drift",
|
||||
severity: "blocker",
|
||||
field: "judgeProbe.behaviorVersion",
|
||||
message: "/api/judge/probe behavior version does not match the repo contract",
|
||||
expected: expectedJudgeProbeBehaviorVersion,
|
||||
observed: observation.judgeProbe.behaviorVersion,
|
||||
});
|
||||
} else if (!observation.judgeProbe.ok) {
|
||||
signals.push({
|
||||
code: "deployment-drift",
|
||||
severity: "blocker",
|
||||
field: "judgeProbe.ok",
|
||||
message: "/api/judge/probe returned a non-ready result",
|
||||
expected: true,
|
||||
observed: observation.judgeProbe.error,
|
||||
});
|
||||
}
|
||||
return signals;
|
||||
}
|
||||
|
||||
function residualSignals(observation: CodeQueueExecutionPlaneObservation): DriftSignal[] {
|
||||
const signals: DriftSignal[] = [];
|
||||
if (observation.residual.composeBackend.present || observation.residual.loopbackPort4222.present) {
|
||||
signals.push({
|
||||
code: "deprecated-compose-residual",
|
||||
severity: "blocker",
|
||||
message: "Deprecated Docker Compose code-queue-backend or old 127.0.0.1:4222 listener is still present",
|
||||
observed: {
|
||||
composeBackendPresent: observation.residual.composeBackend.present,
|
||||
loopbackPort4222Present: observation.residual.loopbackPort4222.present,
|
||||
},
|
||||
});
|
||||
}
|
||||
if (!observation.residual.composeBackend.ok) {
|
||||
signals.push({
|
||||
code: "deprecated-compose-residual-observation-gap",
|
||||
severity: "warning",
|
||||
message: "Could not observe Docker Compose residual state",
|
||||
observed: observation.residual.composeBackend.error,
|
||||
});
|
||||
}
|
||||
if (!observation.residual.loopbackPort4222.ok) {
|
||||
signals.push({
|
||||
code: "deprecated-port-observation-gap",
|
||||
severity: "warning",
|
||||
message: "Could not observe local listener state for 127.0.0.1:4222",
|
||||
observed: observation.residual.loopbackPort4222.error,
|
||||
});
|
||||
}
|
||||
return signals;
|
||||
}
|
||||
|
||||
function compactDeployment(deployment: DeploymentObservation): Record<string, unknown> {
|
||||
return {
|
||||
name: deployment.name,
|
||||
observed: deployment.observed,
|
||||
role: deployment.env.serviceRole,
|
||||
expectedRole: deployment.expectedRole,
|
||||
schedulerEnabled: deployment.env.schedulerEnabled,
|
||||
expectedSchedulerEnabled: deployment.expectedSchedulerEnabled,
|
||||
readyReplicas: deployment.replicas.ready,
|
||||
image: deployment.image,
|
||||
nodeSelector: deployment.nodeSelector,
|
||||
deployCommit: deployment.env.codeQueueDeployCommit ?? deployment.env.unideskDeployCommit ?? deployment.annotations.deployCommit,
|
||||
requestedCommit: deployment.env.codeQueueDeployRequestedCommit ?? deployment.env.unideskDeployRequestedCommit ?? deployment.annotations.deployRequestedCommit,
|
||||
repoHostPath: deployment.repoHostPath,
|
||||
error: deployment.error,
|
||||
};
|
||||
}
|
||||
|
||||
function compactJudgeProbe(probe: JudgeProbeObservation, full: boolean): Record<string, unknown> {
|
||||
return {
|
||||
ok: probe.ok,
|
||||
attempted: probe.attempted,
|
||||
behaviorVersion: probe.behaviorVersion,
|
||||
expectedBehaviorVersion: probe.expectedBehaviorVersion,
|
||||
configured: probe.configured,
|
||||
model: probe.model,
|
||||
hits: probe.hits,
|
||||
total: probe.total,
|
||||
hitRate: probe.hitRate,
|
||||
serviceProxyPath: probe.serviceProxyPath,
|
||||
error: probe.error,
|
||||
...(full ? { raw: probe.raw } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function evaluateObservation(observation: CodeQueueExecutionPlaneObservation, options: ExecutionPlaneOptions): Record<string, unknown> {
|
||||
const formalSignals = observation.deployments.flatMap(deploymentFormalSignals);
|
||||
const deploymentDriftSignals = driftSignals(observation);
|
||||
const deprecatedResidualSignals = residualSignals(observation);
|
||||
const guardSignals: DriftSignal[] = observation.guard.status === "pass"
|
||||
? []
|
||||
: [{
|
||||
code: "d601-k3s-guard-blocked",
|
||||
severity: "blocker",
|
||||
message: observation.guard.summary,
|
||||
expected: { kubeconfig: d601NativeKubeconfig, node: d601RequiredNodeName },
|
||||
observed: { status: observation.guard.status, nodes: observation.guard.nodeNames },
|
||||
}];
|
||||
const signals = [...guardSignals, ...formalSignals, ...deploymentDriftSignals, ...deprecatedResidualSignals];
|
||||
const blockers = signals.filter((signal) => signal.severity === "blocker");
|
||||
const warnings = signals.filter((signal) => signal.severity === "warning");
|
||||
const status: PlaneStatus = blockers.length > 0 ? "blocked" : warnings.length > 0 ? "degraded" : "ready";
|
||||
const formal = formalSignals.length === 0 && observation.deployments.length === expectedDeployments.length;
|
||||
const uniqueDigests = uniqueStrings(observation.pods.map((pod) => pod.digest));
|
||||
const uniqueCommits = uniqueStrings(observation.deployments.flatMap((deployment) => Object.values(deploymentCommitMarkers(deployment))));
|
||||
const result: Record<string, unknown> = {
|
||||
ok: status === "ready",
|
||||
surface: "code-queue-execution-plane",
|
||||
checkedAt: observation.checkedAt,
|
||||
devTestClass: "live-read",
|
||||
mutation: false,
|
||||
namespace: observation.namespace,
|
||||
kubeconfig: observation.kubeconfig,
|
||||
status,
|
||||
runnerDisposition: status === "ready" ? "ready" : "infra-blocked",
|
||||
summary: {
|
||||
formalExecutionPlane: formal,
|
||||
deploymentDrift: deploymentDriftSignals.some((signal) => signal.severity === "blocker"),
|
||||
deprecatedComposeResidual: deprecatedResidualSignals.some((signal) => signal.code === "deprecated-compose-residual"),
|
||||
blockerCount: blockers.length,
|
||||
warningCount: warnings.length,
|
||||
deploymentsObserved: observation.deployments.filter((deployment) => deployment.observed).length,
|
||||
readyPodCount: observation.pods.filter((pod) => pod.ready).length,
|
||||
artifactDigest: uniqueDigests.length === 1 ? uniqueDigests[0] : null,
|
||||
deploymentCommit: uniqueCommits.length === 1 ? uniqueCommits[0] : null,
|
||||
mountedWorktreeHead: observation.worktree.head,
|
||||
judgeProbeBehaviorVersion: observation.judgeProbe.behaviorVersion,
|
||||
},
|
||||
guard: {
|
||||
status: observation.guard.status,
|
||||
kubeconfig: observation.guard.kubeconfig,
|
||||
nodeNames: observation.guard.nodeNames,
|
||||
requiredNodeName: observation.guard.requiredNodeName,
|
||||
requiredNodePresent: observation.guard.requiredNodePresent,
|
||||
summary: observation.guard.summary,
|
||||
},
|
||||
executionPlane: {
|
||||
formal,
|
||||
expectedDeployments: expectedDeployments.map((deployment) => deployment.name),
|
||||
deployments: observation.deployments.map(compactDeployment),
|
||||
},
|
||||
drift: {
|
||||
status: deploymentDriftSignals.some((signal) => signal.severity === "blocker") ? "deployment-drift" : "none",
|
||||
signalCount: deploymentDriftSignals.length,
|
||||
signals: deploymentDriftSignals.slice(0, options.full ? undefined : 5),
|
||||
omittedSignals: options.full ? 0 : Math.max(0, deploymentDriftSignals.length - 5),
|
||||
comparedFields: ["deployment env/annotation commit", "artifact digest", "mounted worktree HEAD", "/api/judge/probe behaviorVersion"],
|
||||
},
|
||||
residual: {
|
||||
status: deprecatedResidualSignals.some((signal) => signal.code === "deprecated-compose-residual") ? "deprecated-compose-residual" : "none",
|
||||
composeBackendPresent: observation.residual.composeBackend.present,
|
||||
loopbackPort4222Present: observation.residual.loopbackPort4222.present,
|
||||
containers: observation.residual.composeBackend.containers,
|
||||
listeners: observation.residual.loopbackPort4222.listeners,
|
||||
},
|
||||
judgeProbe: compactJudgeProbe(observation.judgeProbe, options.full),
|
||||
blockers,
|
||||
warnings,
|
||||
commands: {
|
||||
refresh: "bun scripts/cli.ts codex execution-plane",
|
||||
full: "bun scripts/cli.ts codex execution-plane --full",
|
||||
raw: "bun scripts/cli.ts codex execution-plane --raw",
|
||||
},
|
||||
disclosure: {
|
||||
defaultView: "compact-low-noise",
|
||||
fullDetailOmitted: !options.full,
|
||||
rawOmitted: !options.raw,
|
||||
expandWith: "bun scripts/cli.ts codex execution-plane --full",
|
||||
rawWith: "bun scripts/cli.ts codex execution-plane --raw",
|
||||
secretValuesPrinted: false,
|
||||
rawKubernetesDeploymentJsonPrinted: false,
|
||||
},
|
||||
};
|
||||
if (options.full) {
|
||||
result.details = {
|
||||
deployments: observation.deployments,
|
||||
pods: observation.pods,
|
||||
services: observation.services,
|
||||
worktree: observation.worktree,
|
||||
commandDiagnostics: observation.commandDiagnostics,
|
||||
};
|
||||
}
|
||||
if (options.raw) result.rawObservation = observation;
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function runCodeQueueExecutionPlane(args: string[], collector: ExecutionPlaneCollector = new LiveExecutionPlaneCollector()): Promise<Record<string, unknown>> {
|
||||
const options = parseExecutionPlaneOptions(args);
|
||||
const observation = await collector.collect(options);
|
||||
return evaluateObservation(observation, options);
|
||||
}
|
||||
|
||||
export async function runCodeQueueExecutionPlaneForTest(args: string[], observation: CodeQueueExecutionPlaneObservation): Promise<Record<string, unknown>> {
|
||||
return runCodeQueueExecutionPlane(args, { collect: () => observation });
|
||||
}
|
||||
Reference in New Issue
Block a user