mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: diagnose code agent by agentrun ids

Browse Source
This commit is contained in:
Codex
2026-06-26 22:15:12 +00:00
parent 5ddb63ef37
commit 8aae34f610
7 changed files with 191 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/platform-infra-observability/apply-status-scripts.ts
+7
View File
@@ -61,7 +61,11 @@ export function compactDiagnoseCodeAgentResult(value: unknown): Record<string, u
mapping: mapping === null ? null : {
mode: mapping.mode ?? null,
businessTraceId: mapping.businessTraceId ?? null,
runId: mapping.runId ?? null,
commandId: mapping.commandId ?? null,
runnerJobId: mapping.runnerJobId ?? null,
otelTraceId: mapping.otelTraceId ?? null,
searchQuery: mapping.searchQuery ?? null,
searchOk: mapping.searchOk ?? null,
searchParseOk: mapping.searchParseOk ?? null,
candidateTraceCount: mapping.candidateTraceCount ?? null,
@@ -143,6 +147,8 @@ export function compactDiagnoseAgentRun(value: unknown): Record<string, unknown>
terminalSource: agentrun.terminalSource ?? null,
latestSeq: agentrun.latestSeq ?? null,
terminalEventType: agentrun.terminalEventType ?? null,
failureKind: agentrun.failureKind ?? null,
observedFailureKinds: limitArray(agentrun.observedFailureKinds, 5),
runnerProviderClassification: agentrun.runnerProviderClassification ?? null,
authority: compactAgentRunAuthority(agentrun.authority),
};
@@ -166,6 +172,7 @@ export function compactAgentRunAuthority(value: unknown): Record<string, unknown
terminalSource: authority.terminalSource ?? null,
terminalEventSeq: authority.terminalEventSeq ?? null,
terminalEventType: authority.terminalEventType ?? null,
failureKind: authority.failureKind ?? null,
latestSeq: authority.latestSeq ?? null,
eventCount: authority.eventCount ?? null,
fallback: authority.fallback ?? null,
scripts/src/platform-infra-observability/diagnose-code-agent-script.ts
+126 -9
View File
@@ -746,14 +746,75 @@ PY
`;
}
function cliArg(value: string): string {
return /^[A-Za-z0-9_./:=@+-]+$/u.test(value) ? value : shQuote(value);
}
function diagnoseSearchFilters(options: DiagnoseCodeAgentOptions): string[] {
const filters: string[] = [];
if (options.businessTraceId !== null) filters.push(`.traceId = ${JSON.stringify(options.businessTraceId)}`);
if (options.runId !== null) filters.push(`.runId = ${JSON.stringify(options.runId)}`);
if (options.commandId !== null) filters.push(`.commandId = ${JSON.stringify(options.commandId)}`);
if (options.runnerJobId !== null) filters.push(`.runnerJobId = ${JSON.stringify(options.runnerJobId)}`);
return filters;
}
function diagnoseSearchMode(options: DiagnoseCodeAgentOptions): string {
if (options.businessTraceId !== null && options.runId === null && options.commandId === null && options.runnerJobId === null) return "business-trace-id";
if (options.traceId !== null) return "trace-id";
return "trace-attribute-query";
}
function buildDiagnoseCodeAgentCommand(target: ObservabilityTarget, options: DiagnoseCodeAgentOptions, full: boolean): string {
const parts = ["bun", "scripts/cli.ts", "platform-infra", "observability", "diagnose-code-agent", "--target", target.id];
if (options.businessTraceId !== null) parts.push("--business-trace-id", options.businessTraceId);
if (options.traceId !== null) parts.push("--trace-id", options.traceId);
if (options.runId !== null) parts.push("--run-id", options.runId);
if (options.commandId !== null) parts.push("--command-id", options.commandId);
if (options.runnerJobId !== null) parts.push("--runner-job-id", options.runnerJobId);
parts.push("--lookback-minutes", String(options.lookbackMinutes), "--candidate-limit", String(options.candidateLimit), "--limit", String(options.limit));
if (full) parts.push("--full");
return parts.map(cliArg).join(" ");
}
function buildDiagnoseSearchCommand(target: ObservabilityTarget, tempoQuery: string | null): string {
const query = tempoQuery ?? "{ .traceId = \"<trc_...>\" }";
return [
"bun",
"scripts/cli.ts",
"platform-infra",
"observability",
"search",
"--target",
target.id,
"--query",
query,
"--lookback-minutes",
"10080",
"--candidate-limit",
"500",
"--limit",
"10",
].map(cliArg).join(" ");
}
export function diagnoseCodeAgentScript(observability: ObservabilityConfig, target: ObservabilityTarget, searchPath: string | null, options: DiagnoseCodeAgentOptions): string {
const proxyPrefix = `/api/v1/namespaces/${target.namespace}/services/http:${observability.traceBackend.serviceName}:http/proxy`;
const searchProxyPath = searchPath === null ? null : `${proxyPrefix}${searchPath}`;
const traceIdLiteral = options.traceId === null ? "None" : JSON.stringify(options.traceId);
const businessTraceIdLiteral = options.businessTraceId === null ? "None" : JSON.stringify(options.businessTraceId);
const runIdLiteral = options.runId === null ? "None" : JSON.stringify(options.runId);
const commandIdLiteral = options.commandId === null ? "None" : JSON.stringify(options.commandId);
const runnerJobIdLiteral = options.runnerJobId === null ? "None" : JSON.stringify(options.runnerJobId);
const searchPathLiteral = searchPath === null ? "None" : JSON.stringify(searchPath);
const searchProxyPathLiteral = searchProxyPath === null ? "None" : JSON.stringify(searchProxyPath);
const businessTraceIdForNext = options.businessTraceId ?? "<trc_...>";
const tempoQuery = (() => {
const filters = diagnoseSearchFilters(options);
return filters.length > 0 ? `{ ${filters.join(" && ")} }` : null;
})();
const searchMode = diagnoseSearchMode(options);
const diagnoseFullCommand = buildDiagnoseCodeAgentCommand(target, options, true);
const searchCommand = buildDiagnoseSearchCommand(target, tempoQuery);
return `
set -u
python3 - <<'PY'
@@ -761,11 +822,18 @@ import collections, json, re, subprocess, time, urllib.parse
BUSINESS_TRACE_ID = ${businessTraceIdLiteral}
TRACE_ID = ${traceIdLiteral}
RUN_ID = ${runIdLiteral}
COMMAND_ID = ${commandIdLiteral}
RUNNER_JOB_ID = ${runnerJobIdLiteral}
TARGET_ID = ${JSON.stringify(target.id)}
SEARCH_MODE = ${JSON.stringify(searchMode)}
SEARCH_QUERY = ${tempoQuery === null ? "None" : JSON.stringify(tempoQuery)}
SEARCH_PATH = ${searchPathLiteral}
SEARCH_PROXY_PATH = ${searchProxyPathLiteral}
TRACE_PROXY_PREFIX = ${JSON.stringify(proxyPrefix)}
TRACE_PATH_TEMPLATE = ${JSON.stringify(observability.probes.traceQueryPathTemplate)}
DIAGNOSE_FULL_COMMAND = ${JSON.stringify(diagnoseFullCommand)}
SEARCH_COMMAND = ${JSON.stringify(searchCommand)}
FULL = ${options.full ? "True" : "False"}
RAW = ${options.raw ? "True" : "False"}
LIMIT = ${options.limit}
@@ -982,6 +1050,28 @@ def classify_terminal_status(status, spans):
return "running"
return "unknown"
def preferred_failure_kind(values):
kinds = []
for value in values:
if value in (None, ""):
continue
text = str(value).strip()
if not text:
continue
if text not in kinds:
kinds.append(text)
if not kinds:
return None
for kind in kinds:
lowered = kind.lower()
if lowered.startswith("provider-") or lowered.startswith("provider_") or "provider-auth" in lowered:
return kind
for kind in kinds:
lowered = kind.lower()
if lowered not in ("infra-failed", "infra_failed", "unknown"):
return kind
return kinds[-1]
def agentrun_authority_summary(identity):
run_id = identity.get("runId") if isinstance(identity, dict) else None
command_id = identity.get("commandId") if isinstance(identity, dict) else None
@@ -1382,7 +1472,7 @@ def http_status_summary(spans):
def agentrun_summary(spans, authority=None):
terminal_spans = []
latest_seq = None
failure_kind = None
failure_kinds = []
failure_message = None
terminal_category = None
seq_keys = ("maxSeq", "traceLastSeq", "endSeq", "seq", "eventSeq", "sourceSeq", "sourceLatestSeq", "latestSeq", "lastSeq")
@@ -1401,7 +1491,7 @@ def agentrun_summary(spans, authority=None):
if not derived and name.startswith("runner_terminal."):
derived = name.split(".", 1)[1]
if attrs.get("failureKind") not in (None, ""):
failure_kind = attrs.get("failureKind")
failure_kinds.append(attrs.get("failureKind"))
if attrs.get("failureMessage") not in (None, ""):
failure_message = attrs.get("failureMessage")
if attrs.get("terminalCategory") not in (None, ""):
@@ -1423,12 +1513,15 @@ def agentrun_summary(spans, authority=None):
if isinstance(authority, dict) and authority.get("terminalStatus") not in (None, ""):
terminal_status = authority.get("terminalStatus")
terminal_event_type = authority.get("terminalEventType") or terminal_event_type
failure_kind = authority.get("failureKind") or failure_kind
if authority.get("failureKind") not in (None, ""):
failure_kinds.append(authority.get("failureKind"))
terminal_source = "agentrun-authority"
failure_kind = preferred_failure_kind(failure_kinds)
runner_classification = classify_terminal_status(terminal_status, spans)
return {
"terminalStatus": terminal_status,
"failureKind": failure_kind,
"observedFailureKinds": failure_kinds,
"failureMessage": failure_message,
"terminalCategory": terminal_category,
"latestSeq": latest_seq,
@@ -1562,6 +1655,20 @@ def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error
"summary": "HWLAB projection stale is suspected because AgentRun is terminal but HWLAB read-model evidence is incomplete.",
"evidence": lag_summary.get("reasons", []),
})
failure_kind = str(agentrun.get("failureKind") or "")
if failure_kind == "provider-auth-failed":
candidates.append({
"code": "provider_auth_failed",
"label": "provider auth failed",
"confidence": 0.96,
"summary": "Provider credential refresh/auth failed before the manager reconciler closed the command; fix provider auth before retrying this Code Agent path.",
"evidence": {
"terminalStatus": agentrun.get("terminalStatus"),
"failureKind": agentrun.get("failureKind"),
"observedFailureKinds": agentrun.get("observedFailureKinds"),
"terminalCategory": agentrun.get("terminalCategory"),
},
})
if agentrun.get("terminalStatus") in ("failed", "error", "timeout", "blocked", "cancelled"):
candidates.append({
"code": "agentrun_terminal_failed",
@@ -1712,7 +1819,11 @@ def resolve_trace():
return TRACE_ID, {
"mode": "trace-id",
"businessTraceId": BUSINESS_TRACE_ID,
"runId": RUN_ID,
"commandId": COMMAND_ID,
"runnerJobId": RUNNER_JOB_ID,
"otelTraceId": TRACE_ID,
"searchQuery": SEARCH_QUERY,
"searchPath": None,
"searchOk": None,
"searchParseOk": None,
@@ -1765,9 +1876,13 @@ def resolve_trace():
if selected_low_confidence and not selected_complete:
selected_rejected_reason = "best candidate is only a single-span Workbench GET/SSE trace; no high-confidence Code Agent trace was found in the scanned window"
mapping = {
"mode": "business-trace-id",
"mode": SEARCH_MODE,
"businessTraceId": BUSINESS_TRACE_ID,
"runId": RUN_ID,
"commandId": COMMAND_ID,
"runnerJobId": RUNNER_JOB_ID,
"otelTraceId": selected_trace_id,
"searchQuery": SEARCH_QUERY,
"searchPath": SEARCH_PATH,
"searchProxyPath": SEARCH_PROXY_PATH,
"searchOk": search_rc == 0,
@@ -1787,7 +1902,7 @@ def resolve_trace():
if RAW:
mapping["rawSearchBody"] = search_parsed if isinstance(search_parsed, dict) else search_body[-12000:]
if selected_trace_id is None:
return None, mapping, "no OTel trace found for business trace"
return None, mapping, "no OTel trace found for diagnosis query"
if selected_rejected_reason is not None:
return None, mapping, selected_rejected_reason
return selected_trace_id, mapping, None
@@ -1803,8 +1918,8 @@ if resolved_trace_id is None:
"facts": [],
},
"next": {
"expandWindow": "bun scripts/cli.ts platform-infra observability diagnose-code-agent --target ${target.id} --business-trace-id ${businessTraceIdForNext} --lookback-minutes 10080 --candidate-limit 500 --full",
"search": "bun scripts/cli.ts platform-infra observability search --target ${target.id} --query '{ .traceId = \\"${businessTraceIdForNext}\\" }' --lookback-minutes 10080 --candidate-limit 500 --limit 10",
"expandWindow": DIAGNOSE_FULL_COMMAND,
"search": SEARCH_COMMAND,
},
}
print(json.dumps(payload, ensure_ascii=False, indent=2))
@@ -1907,6 +2022,8 @@ payload = {
"agentrun": {
"terminalStatus": agentrun.get("terminalStatus"),
"terminalSource": agentrun.get("terminalSource"),
"failureKind": agentrun.get("failureKind"),
"observedFailureKinds": agentrun.get("observedFailureKinds"),
"latestSeq": agentrun.get("latestSeq"),
"terminalEventType": agentrun.get("terminalEventType"),
"runnerProviderClassification": agentrun.get("runnerProviderClassification"),
@@ -1931,7 +2048,7 @@ payload = {
"spanNameCounts": flat["spanNameCounts"][:12],
"evidence": evidence,
"next": {
"diagnoseFull": "bun scripts/cli.ts platform-infra observability diagnose-code-agent --target ${target.id} --business-trace-id ${businessTraceIdForNext} --full",
"diagnoseFull": DIAGNOSE_FULL_COMMAND,
"traceSummary": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --limit 80" % resolved_trace_id,
"traceReads": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep trace_events_read --limit 20 --full" % resolved_trace_id,
"turnStatus": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep turn_status_read --limit 20 --full" % resolved_trace_id,
scripts/src/platform-infra-observability/manifest.ts
+3
View File
@@ -207,6 +207,9 @@ export function buildDiagnoseCommand(target: ObservabilityTarget, options: Diagn
const parts = ["bun", "scripts/cli.ts", "platform-infra", "observability", "diagnose-code-agent", "--target", target.id];
if (options.businessTraceId !== null) parts.push("--business-trace-id", options.businessTraceId);
if (options.traceId !== null) parts.push("--trace-id", options.traceId);
if (options.runId !== null) parts.push("--run-id", options.runId);
if (options.commandId !== null) parts.push("--command-id", options.commandId);
if (options.runnerJobId !== null) parts.push("--runner-job-id", options.runnerJobId);
parts.push("--lookback-minutes", String(options.lookbackMinutes), "--candidate-limit", String(options.candidateLimit), "--limit", String(options.limit));
if (full) parts.push("--full");
return parts.map(cliArg).join(" ");
scripts/src/platform-infra-observability/options.ts
+26 -2
View File
@@ -39,6 +39,7 @@ export function observabilityHelp(): Record<string, unknown> {
"bun scripts/cli.ts platform-infra observability search --target D601 --grep 'no rollout found' [--lookback-minutes 360] [--candidate-limit 80] [--limit 20] [--full|--raw]",
"bun scripts/cli.ts platform-infra observability search --target D601 --path /v1/workbench/sessions --status 502 [--lookback-minutes 120] [--full|--raw]",
"bun scripts/cli.ts platform-infra observability diagnose-code-agent --target D601 --business-trace-id <trc_...> [--full|--raw]",
"bun scripts/cli.ts platform-infra observability diagnose-code-agent --target D601 --run-id <run_...> [--command-id <cmd_...>] [--runner-job-id <rjob_...>] [--full|--raw]",
],
boundary: "Prometheus remains the metrics source; this command owns only platform-infra OTel Collector, trace backend readiness, and trace lookup.",
};
@@ -217,6 +218,9 @@ export function parseDiagnoseCodeAgentOptions(args: string[]): DiagnoseCodeAgent
const commonArgs: string[] = [];
let businessTraceId: string | null = null;
let traceId: string | null = null;
let runId: string | null = null;
let commandId: string | null = null;
let runnerJobId: string | null = null;
let limit = 40;
let candidateLimit = 300;
let lookbackMinutes = 10080;
@@ -234,6 +238,24 @@ export function parseDiagnoseCodeAgentOptions(args: string[]): DiagnoseCodeAgent
if (!/^[0-9a-fA-F]{32}$/u.test(value)) throw new Error(`${arg} must be a 32-character OpenTelemetry trace id encoded as hex`);
traceId = value.toLowerCase();
index += 1;
} else if (arg === "--run-id" || arg === "--run") {
const value = args[index + 1];
if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
if (!/^run_[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`${arg} must look like run_<id>`);
runId = value;
index += 1;
} else if (arg === "--command-id" || arg === "--command") {
const value = args[index + 1];
if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
if (!/^cmd_[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`${arg} must look like cmd_<id>`);
commandId = value;
index += 1;
} else if (arg === "--runner-job-id" || arg === "--runner-job" || arg === "--runnerjob") {
const value = args[index + 1];
if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
if (!/^rjob_[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`${arg} must look like rjob_<id>`);
runnerJobId = value;
index += 1;
} else if (arg === "--limit") {
const value = args[index + 1];
if (value === undefined || value.startsWith("--")) throw new Error("--limit requires a value");
@@ -263,6 +285,8 @@ export function parseDiagnoseCodeAgentOptions(args: string[]): DiagnoseCodeAgent
}
}
}
if (businessTraceId === null && traceId === null) throw new Error("observability diagnose-code-agent requires --business-trace-id <trc_...> or --trace-id <otelTraceId>");
return { ...parseCommonOptions(commonArgs), businessTraceId, traceId, limit, candidateLimit, lookbackMinutes };
if (businessTraceId === null && traceId === null && runId === null && commandId === null && runnerJobId === null) {
throw new Error("observability diagnose-code-agent requires --business-trace-id <trc_...>, --trace-id <otelTraceId>, --run-id <run_...>, --command-id <cmd_...>, or --runner-job-id <rjob_...>");
}
return { ...parseCommonOptions(commonArgs), businessTraceId, traceId, runId, commandId, runnerJobId, limit, candidateLimit, lookbackMinutes };
}
scripts/src/platform-infra-observability/render.ts
+25 -3
View File
@@ -141,18 +141,34 @@ export function businessTraceIdFromSearchText(value: string | null): string | nu
return match?.[0] ?? null;
}
function diagnoseTraceSearchFilters(options: DiagnoseCodeAgentOptions): string[] {
const filters: string[] = [];
if (options.businessTraceId !== null) filters.push(`.traceId = ${JSON.stringify(options.businessTraceId)}`);
if (options.runId !== null) filters.push(`.runId = ${JSON.stringify(options.runId)}`);
if (options.commandId !== null) filters.push(`.commandId = ${JSON.stringify(options.commandId)}`);
if (options.runnerJobId !== null) filters.push(`.runnerJobId = ${JSON.stringify(options.runnerJobId)}`);
return filters;
}
function diagnoseTraceSearchMode(options: DiagnoseCodeAgentOptions): string {
if (options.businessTraceId !== null && options.runId === null && options.commandId === null && options.runnerJobId === null) return "business-trace-id";
return "trace-attribute-query";
}
export async function diagnoseCodeAgent(config: UniDeskConfig, options: DiagnoseCodeAgentOptions): Promise<Record<string, unknown> | RenderedCliResult> {
const observability = readObservabilityConfig();
const target = resolveTarget(observability, options.targetId);
const endSeconds = Math.floor(Date.now() / 1000);
const startSeconds = endSeconds - (options.lookbackMinutes * 60);
let searchPath: string | null = null;
if (options.traceId === null && options.businessTraceId !== null) {
const searchFilters = diagnoseTraceSearchFilters(options);
const tempoQuery = searchFilters.length > 0 ? `{ ${searchFilters.join(" && ")} }` : null;
if (options.traceId === null && tempoQuery !== null) {
const params = new URLSearchParams({
start: String(startSeconds),
end: String(endSeconds),
limit: String(options.candidateLimit),
q: `{ .traceId = "${options.businessTraceId}" }`,
q: tempoQuery,
});
searchPath = `/api/search?${params.toString()}`;
}
@@ -164,6 +180,11 @@ export async function diagnoseCodeAgent(config: UniDeskConfig, options: Diagnose
service: observability.traceBackend.serviceName,
businessTraceId: options.businessTraceId,
traceId: options.traceId,
runId: options.runId,
commandId: options.commandId,
runnerJobId: options.runnerJobId,
mode: diagnoseTraceSearchMode(options),
tempoQuery,
path: searchPath,
lookbackMinutes: options.lookbackMinutes,
candidateLimit: options.candidateLimit,
@@ -530,6 +551,7 @@ export function renderDiagnoseCodeAgentTable(input: {
"",
"Identity:",
` businessTraceId=${textValue(mapping?.businessTraceId ?? input.query.businessTraceId)} otelTraceId=${traceId}`,
` queryMode=${textValue(mapping?.mode ?? input.query.mode)} tempoQuery=${shortenMiddle(textValue(mapping?.searchQuery ?? input.query.tempoQuery), 80)}`,
` runId=${textValue(identity?.runId)} commandId=${textValue(identity?.commandId)} runnerJobId=${textValue(identity?.runnerJobId)} runnerId=${textValue(identity?.runnerId)}`,
` backendProfile=${textValue(identity?.backendProfile)} sourceCommit=${shortenMiddle(textValue(identity?.sourceCommit), 20)}`,
"",
@@ -541,7 +563,7 @@ export function renderDiagnoseCodeAgentTable(input: {
"",
"Summary:",
` target=${input.target.id} spanCount=${textValue(input.result.spanCount)} servicePath=${joinValues(input.result.servicePath, 60)}`,
` agentrun=${textValue(agentrun?.terminalStatus)} source=${textValue(agentrun?.terminalSource)} authority=${shortenEnd(JSON.stringify((asPlainRecord(agentrun?.authority) ?? {})), 120)}`,
` agentrun=${textValue(agentrun?.terminalStatus)} failureKind=${textValue(agentrun?.failureKind)} source=${textValue(agentrun?.terminalSource)} authority=${shortenEnd(JSON.stringify((asPlainRecord(agentrun?.authority) ?? {})), 120)}`,
` readModel=${shortenEnd(JSON.stringify(input.result.hwlabReadModel ?? null), 140)}`,
];
const next = asPlainRecord(input.result.next);
scripts/src/platform-infra-observability/types.ts
+3
View File
@@ -157,6 +157,9 @@ export interface SearchOptions extends CommonOptions {
export interface DiagnoseCodeAgentOptions extends CommonOptions {
businessTraceId: string | null;
traceId: string | null;
runId: string | null;
commandId: string | null;
runnerJobId: string | null;
limit: number;
candidateLimit: number;
lookbackMinutes: number;
scripts/src/platform-infra/entry.ts
+1
View File
@@ -359,6 +359,7 @@ export function platformInfraHelp(): unknown {
"bun scripts/cli.ts platform-infra observability trace --target D601 --trace-id <traceId> [--grep text] [--limit 40] [--full|--raw]",
"bun scripts/cli.ts platform-infra observability search --target D601 --grep 'no rollout found' [--lookback-minutes 360] [--candidate-limit 80] [--limit 20]",
"bun scripts/cli.ts platform-infra observability diagnose-code-agent --target D601 --business-trace-id <trc_...> [--full|--raw]",
"bun scripts/cli.ts platform-infra observability diagnose-code-agent --target D601 --run-id <run_...> [--command-id <cmd_...>] [--runner-job-id <rjob_...>]",
],
description: "Operate YAML-controlled platform-infra services such as Sub2API, LangBot, n8n, WeChat archive workflows and OpenTelemetry tracing. Public services use PK01 Caddy+FRP rather than Kubernetes Ingress, NodePort, or LoadBalancer.",
target,