Merge remote-tracking branch 'origin/master' into fix/1846-artificer-gpt-pika
This commit is contained in:
+15
-4
@@ -190,8 +190,12 @@ const SSH_OPERATION_HELP: Record<SshHelpScope, SshOperationHelp> = {
|
||||
},
|
||||
exec: {
|
||||
group: "process",
|
||||
description: "在 k3s workload route 中运行进程,并可显式透传 stdin 或指定容器内 cwd。",
|
||||
usage: ["trans <provider>:k3s:<namespace>:<workload>[:<container>] exec [--cwd /path] [--stdin] -- <command> [args...]"],
|
||||
description: "运行单个远端进程;host/workspace 的 cwd 写入 route,k3s workload 用 --cwd 指定容器内目录。",
|
||||
usage: [
|
||||
"trans <provider>:/absolute/workspace exec <command> [args...]",
|
||||
"trans <provider>:k3s:<namespace>:<workload>[:<container>] exec [--cwd /path] [--stdin] -- <command> [args...]",
|
||||
],
|
||||
notes: ["host/workspace 不接受 exec --cwd;改写 route,例如 trans D601:/workspace exec git status。"],
|
||||
},
|
||||
git: {
|
||||
group: "process",
|
||||
@@ -221,8 +225,9 @@ const SSH_OPERATION_HELP: Record<SshHelpScope, SshOperationHelp> = {
|
||||
},
|
||||
bash: {
|
||||
group: "shell",
|
||||
description: "仅在需要 pipefail、数组、[[ ]] 等 Bash 语法时执行远端 Bash。",
|
||||
usage: ["trans <route> bash <<'BASH'", "trans <route> bash -- '<bash command>'"],
|
||||
description: "仅在需要 pipefail、数组、[[ ]] 等 Bash 语法时执行远端 Bash;cwd 由 route 选择。",
|
||||
usage: ["trans <provider>:/absolute/workspace bash <<'BASH'", "trans <provider>:k3s:<namespace>:<workload>[/cwd] bash <<'BASH'", "trans <route> bash -- '<bash command>'"],
|
||||
notes: ["host/workspace 不接受 bash --cwd;将绝对目录写入 route。"],
|
||||
},
|
||||
py: {
|
||||
group: "shell",
|
||||
@@ -327,6 +332,11 @@ export function sshHelp(scope: SshHelpScope | undefined = undefined): unknown {
|
||||
"远端文本读取优先 cat/rg,修改优先 apply-patch;upload/download 仅用于必要的二进制或生成物。",
|
||||
"普通 trans 短连接上限保持 60s;长流程使用受控 submit-and-poll 入口。",
|
||||
],
|
||||
cwdSemantics: {
|
||||
hostWorkspace: `${entrypoint} <provider>:/absolute/workspace exec|sh|bash ...`,
|
||||
k3sWorkload: `${entrypoint} <provider>:k3s:<namespace>:<workload>[:<container>] exec --cwd /absolute/container/path -- <command>`,
|
||||
rule: "host/workspace cwd 属于 route;k3s 容器 cwd 可由 route 后缀或 exec --cwd 指定。",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -343,6 +353,7 @@ export function renderSshHelpText(scope: SshHelpScope | undefined = undefined):
|
||||
`patch: ${SSH_OPERATION_GROUPS.patch.join(" | ")}`,
|
||||
`transfer: ${SSH_OPERATION_GROUPS.transfer.join(" | ")}`,
|
||||
`scoped help: ${entrypoint} --help <operation>`,
|
||||
`cwd: host/workspace uses <provider>:/absolute/workspace; k3s exec uses --cwd /absolute/container/path`,
|
||||
"boundary: route only selects the target; use sh/bash/ps/cmd explicitly for shell syntax.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { resolveSentinelChildJson } from "./hwlab-node-web-sentinel-cicd-shared";
|
||||
import { sentinelDeliveryStatus, type SentinelCicdState } from "./hwlab-node-web-sentinel-cicd";
|
||||
|
||||
describe("sentinel CI/CD child JSON recovery", () => {
|
||||
test("recovers remote probe JSON from trans truncation dump", () => {
|
||||
@@ -37,3 +38,22 @@ describe("sentinel CI/CD child JSON recovery", () => {
|
||||
expect(resolved.diagnostics.source).toBe("dump");
|
||||
});
|
||||
});
|
||||
|
||||
test("sentinel delivery status distinguishes cadence convergence without blocking", () => {
|
||||
const commit = "f87cd3ac8fd2494f9dfed51e7d8ececd3fc42939";
|
||||
const state = { sourceHead: { commit } } as unknown as SentinelCicdState;
|
||||
const base = {
|
||||
sourceMirror: { ok: true },
|
||||
registry: { probe: { present: true } },
|
||||
gitMirror: { ok: true },
|
||||
gitops: { ok: true, sourceCommit: commit, schedule: "0 */2 * * *", revision: "gitops-revision" },
|
||||
argo: { ok: true, syncStatus: "Synced", healthStatus: "Healthy" },
|
||||
runtime: { ok: true, probe: { deployment: { sourceCommit: commit } } },
|
||||
};
|
||||
const converging = sentinelDeliveryStatus(state, { ...base, cadence: { ok: false, probe: { schedule: "0 * * * *" } } }, "pipeline-run");
|
||||
expect(converging.outcome).toBe("converging");
|
||||
expect(converging.blocking).toBe(false);
|
||||
const converged = sentinelDeliveryStatus(state, { ...base, cadence: { ok: true, probe: { schedule: "0 */2 * * *" } } }, "pipeline-run");
|
||||
expect(converged.outcome).toBe("converged");
|
||||
expect(converged.desiredSourceCommit).toBe(commit);
|
||||
});
|
||||
|
||||
@@ -102,6 +102,7 @@ export type WebProbeSentinelOptions =
|
||||
readonly wait: boolean;
|
||||
readonly timeoutSeconds: number;
|
||||
readonly quickVerify: boolean;
|
||||
readonly localOnly: boolean;
|
||||
}
|
||||
| {
|
||||
readonly kind: "report";
|
||||
@@ -659,6 +660,7 @@ export function renderControlPlaneResult(result: Record<string, unknown>): strin
|
||||
const gitops = record(result.gitops);
|
||||
const argo = record(result.argo);
|
||||
const validation = record(result.validation);
|
||||
const deliveryStatus = record(result.deliveryStatus);
|
||||
const observability = record(result.observability);
|
||||
const observed = record(result.observed);
|
||||
const sourceMirrorSync = record(result.sourceMirrorSync);
|
||||
@@ -751,6 +753,11 @@ export function renderControlPlaneResult(result: Record<string, unknown>): strin
|
||||
"",
|
||||
renderObservedStatus(observed),
|
||||
"",
|
||||
Object.keys(deliveryStatus).length === 0 ? "DELIVERY_STATUS\n-" : table(
|
||||
["OUTCOME", "DESIRED_COMMIT", "RUNTIME_COMMIT", "DESIRED_SCHEDULE", "RUNTIME_SCHEDULE", "ARGO", "READY", "CONVERGING", "BLOCKING"],
|
||||
[[deliveryStatus.outcome, short(deliveryStatus.desiredSourceCommit), short(deliveryStatus.runtimeSourceCommit), deliveryStatus.desiredSchedule ?? "-", deliveryStatus.runtimeSchedule ?? "-", `${deliveryStatus.argoSyncStatus ?? "-"}/${deliveryStatus.argoHealthStatus ?? "-"}`, deliveryStatus.runtimeReady, deliveryStatus.converging, deliveryStatus.blocking]],
|
||||
),
|
||||
"",
|
||||
Object.keys(statusDiagnosis).length === 0 ? "STATUS_DIAGNOSIS\n-" : [
|
||||
"STATUS_DIAGNOSIS",
|
||||
table(["CODE", "PHASE", "PIPELINERUN", "SOURCE", "REGISTRY", "GIT_MIRROR", "GITOPS", "ARGO", "RUNTIME"], [[
|
||||
@@ -965,8 +972,8 @@ export function renderAsyncJobResult(result: Record<string, unknown>): string {
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function rendered(ok: boolean, command: string, text: string): RenderedCliResult {
|
||||
return { ok, command, renderedText: `${text.trimEnd()}\n`, contentType: "text/plain" };
|
||||
export function rendered(ok: boolean, command: string, text: string, projection?: Record<string, unknown>): RenderedCliResult {
|
||||
return { ok, command, renderedText: `${text.trimEnd()}\n`, contentType: "text/plain", ...(projection === undefined ? {} : { projection }) };
|
||||
}
|
||||
|
||||
export function sentinelProgressEvent(event: string, payload: Record<string, unknown>): void {
|
||||
|
||||
@@ -168,7 +168,16 @@ export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options:
|
||||
if (blocked !== null) return blocked;
|
||||
}
|
||||
requireSentinelIdForRegistry(spec, options.sentinelId, `web-probe sentinel ${options.kind}`);
|
||||
const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds, sentinelSourceResolveMode(options), sentinelSourceOverrideFromOptions(options));
|
||||
const localCommit = options.kind === "validate" && options.localOnly
|
||||
? runCommand(["git", "rev-parse", "HEAD"], repoRoot, { timeoutMs: 5_000 }).stdout.trim()
|
||||
: null;
|
||||
const localSourceOverride = localCommit === null ? null : {
|
||||
commit: localCommit,
|
||||
stageRef: null,
|
||||
mirrorCommit: localCommit,
|
||||
sourceAuthority: "gitea-snapshot" as const,
|
||||
};
|
||||
const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds, sentinelSourceResolveMode(options), localSourceOverride ?? sentinelSourceOverrideFromOptions(options));
|
||||
if (options.kind === "image") return runSentinelImage(state, options);
|
||||
if (options.kind === "control-plane") return runSentinelControlPlane(state, options);
|
||||
if (options.kind === "publish") return runSentinelPublishCurrent(state, options);
|
||||
@@ -229,13 +238,17 @@ function runSentinelControlPlane(state: SentinelCicdState, options: Extract<WebP
|
||||
if (!options.wait) return renderAsyncSentinelJob(state, "control-plane", options.action, options.timeoutSeconds);
|
||||
return runSentinelControlPlaneConfirmed(state, options);
|
||||
}
|
||||
const observed = options.action === "status" ? collectSentinelObservedStatus(state, options.timeoutSeconds) : null;
|
||||
const observed = options.action === "status"
|
||||
? options.wait
|
||||
? waitForSentinelObservedStatus(state, options.timeoutSeconds)
|
||||
: collectSentinelObservedStatus(state, options.timeoutSeconds)
|
||||
: null;
|
||||
const observedReady = options.action !== "status" || sentinelObservedReady(record(observed));
|
||||
const observedWarnings = options.action === "status" ? sentinelObservedWarnings(record(observed)) : [];
|
||||
const pipelineRun = sentinelPipelineRunName(state, options.rerun);
|
||||
const statusDiagnosis = options.action === "status" && !observedReady ? sentinelObservedStatusDiagnosis(state, observed, pipelineRun) : null;
|
||||
const result = {
|
||||
ok: state.configReady && state.sourceHead.ok && observedReady,
|
||||
ok: state.configReady && state.sourceHead.ok && (options.action === "status" || observedReady),
|
||||
command,
|
||||
node: state.spec.nodeId,
|
||||
lane: state.spec.lane,
|
||||
@@ -283,17 +296,18 @@ function runSentinelControlPlane(state: SentinelCicdState, options: Extract<WebP
|
||||
cicd: state.cicd,
|
||||
}),
|
||||
observed,
|
||||
deliveryStatus: options.action === "status" ? sentinelDeliveryStatus(state, record(observed), pipelineRun) : null,
|
||||
statusDiagnosis,
|
||||
drillDown: controlPlaneDrillDown(state, pipelineRun, null),
|
||||
warnings: mergeWarnings(observedWarnings, record(statusDiagnosis).warning),
|
||||
blocker: observedReady
|
||||
blocker: options.action === "status" || observedReady
|
||||
? null
|
||||
: record(statusDiagnosis).blocker ?? { code: "sentinel-control-plane-observed-not-ready", reason: "one or more source, registry, GitOps, Argo, runtime or cadence checks did not pass", valuesRedacted: true },
|
||||
recoveryNext: record(statusDiagnosis).recoveryNext ?? null,
|
||||
next: controlPlaneNext(state, options.action),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return rendered(result.ok, command, renderControlPlaneResult(result));
|
||||
return rendered(result.ok, command, renderControlPlaneResult(result), options.action === "status" ? record(result.deliveryStatus) : undefined);
|
||||
}
|
||||
|
||||
function runSentinelPublishCurrent(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "publish" }>): RenderedCliResult {
|
||||
@@ -1924,7 +1938,7 @@ function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds:
|
||||
gitops,
|
||||
argo: probeArgoApplication(state, timeoutSeconds, effectiveExpectation.gitopsRevision),
|
||||
runtime: probeRuntimeObjects(state, timeoutSeconds, effectiveExpectation.runtimeImage),
|
||||
cadence: probeCadenceCronJob(state, timeoutSeconds),
|
||||
cadence: probeCadenceCronJob(state, timeoutSeconds, nonEmptyString(gitops.schedule)),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1934,7 +1948,8 @@ function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds:
|
||||
let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
|
||||
let polls = 1;
|
||||
while (!sentinelObservedReady(observed) && Date.now() - startedAt < timeoutMs) {
|
||||
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
||||
const pollSeconds = numberAtNullable(state.cicd, "confirmWait.pollSeconds") ?? 2;
|
||||
runCommand(["sleep", String(pollSeconds)], repoRoot, { timeoutMs: (pollSeconds + 1) * 1_000 });
|
||||
observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
|
||||
polls += 1;
|
||||
}
|
||||
@@ -1964,6 +1979,35 @@ function sentinelObservedReady(value: Record<string, unknown> | SentinelObserved
|
||||
&& record(observed.cadence).ok === true;
|
||||
}
|
||||
|
||||
export function sentinelDeliveryStatus(state: SentinelCicdState, observed: Record<string, unknown>, pipelineRun: string): Record<string, unknown> {
|
||||
const gitops = record(observed.gitops);
|
||||
const runtimeDeployment = record(record(observed.runtime).probe).deployment;
|
||||
const cadenceProbe = record(record(observed.cadence).probe);
|
||||
const argo = record(observed.argo);
|
||||
const desiredSourceCommit = nonEmptyString(gitops.sourceCommit) ?? state.sourceHead.commit;
|
||||
const runtimeSourceCommit = nonEmptyString(record(runtimeDeployment).sourceCommit);
|
||||
const desiredSchedule = nonEmptyString(gitops.schedule);
|
||||
const runtimeSchedule = nonEmptyString(cadenceProbe.schedule);
|
||||
const converging = desiredSourceCommit !== runtimeSourceCommit || desiredSchedule !== runtimeSchedule || !sentinelObservedReady(observed);
|
||||
return {
|
||||
desiredSourceCommit,
|
||||
runtimeSourceCommit,
|
||||
pipelineRun,
|
||||
gitopsRevision: gitops.revision ?? null,
|
||||
argoSyncStatus: argo.syncStatus ?? null,
|
||||
argoHealthStatus: argo.healthStatus ?? null,
|
||||
runtimeReady: record(observed.runtime).ok === true,
|
||||
desiredSchedule,
|
||||
runtimeSchedule,
|
||||
converging,
|
||||
warning: converging ? "GitOps 期望状态尚未完全收敛;这是只读、非阻塞 warning。" : null,
|
||||
blocking: false,
|
||||
outcome: converging ? "converging" : "converged",
|
||||
mutation: false,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function sentinelObservedWarnings(value: Record<string, unknown> | SentinelObservedStatus | null): string[] {
|
||||
const observed = record(value);
|
||||
const argo = record(observed.argo);
|
||||
@@ -2253,6 +2297,8 @@ function probeGitopsRuntimeManifest(state: SentinelCicdState, timeoutSeconds: nu
|
||||
const revision = /^[0-9a-f]{40}$/iu.test(revisionLine?.trim() ?? "") ? revisionLine.trim() : null;
|
||||
const manifest = manifestLines.join("\n");
|
||||
const image = nonEmptyString(manifest.match(/image:\s*([^,\s}\]]+)/u)?.[1]);
|
||||
const sourceCommit = nonEmptyString(manifest.match(/unidesk\.ai\/source-commit:\s*["']?([0-9a-f]{40})/iu)?.[1]);
|
||||
const schedule = nonEmptyString(manifest.match(/\bschedule:\s*["']?([^,"'\r\n#]+)/u)?.[1]?.trim());
|
||||
const imageMatchesRepository = image !== null && image.startsWith(`${state.image.repository}@sha256:`);
|
||||
const compact = compactCommand(result);
|
||||
return {
|
||||
@@ -2261,6 +2307,8 @@ function probeGitopsRuntimeManifest(state: SentinelCicdState, timeoutSeconds: nu
|
||||
branch,
|
||||
manifestPath,
|
||||
image,
|
||||
sourceCommit,
|
||||
schedule,
|
||||
imageMatchesRepository,
|
||||
result: { ...compact, stdoutPreview: `${revision ?? "-"} ${image ?? "-"}` },
|
||||
valuesRedacted: true,
|
||||
@@ -2301,9 +2349,10 @@ function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number, e
|
||||
"const ready = Number(dep?.status?.readyReplicas ?? 0);",
|
||||
"const updated = Number(dep?.status?.updatedReplicas ?? 0);",
|
||||
"const image = dep?.spec?.template?.spec?.containers?.[0]?.image ?? null;",
|
||||
"const sourceCommit = dep?.spec?.template?.metadata?.annotations?.['unidesk.ai/source-commit'] ?? dep?.metadata?.annotations?.['unidesk.ai/source-commit'] ?? null;",
|
||||
"const imageMatches = expectedImage === null || image === expectedImage;",
|
||||
"const payload = {",
|
||||
" deployment: { present: deploymentPresent, desiredReplicas: desired, readyReplicas: ready, updatedReplicas: updated, image, expectedImage, imageMatches },",
|
||||
" deployment: { present: deploymentPresent, desiredReplicas: desired, readyReplicas: ready, updatedReplicas: updated, image, expectedImage, imageMatches, sourceCommit },",
|
||||
" service: { present: rc('svc') === 0 },",
|
||||
" pvc: { present: rc('pvc') === 0, phase: json('pvc')?.status?.phase ?? null },",
|
||||
" configMap: { present: rc('cm') === 0 },",
|
||||
@@ -2320,7 +2369,7 @@ function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number, e
|
||||
return { ok: result.exitCode === 0 && probe?.ok === true, probe, stdoutRecovery: probeResolution.diagnostics, result: compactCommand(result) };
|
||||
}
|
||||
|
||||
function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
||||
function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number, desiredSchedule: string | null = null): Record<string, unknown> {
|
||||
const expected = state.manifests.find((item) => item.kind === "CronJob") ?? null;
|
||||
if (expected === null) {
|
||||
return { ok: true, skipped: true, reason: "targetValidation.cadenceScheduler.disabled", valuesRedacted: true };
|
||||
@@ -2329,7 +2378,7 @@ function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number):
|
||||
const spec = record(expected.spec);
|
||||
const namespace = stringAt(metadata, "namespace");
|
||||
const name = stringAt(metadata, "name");
|
||||
const expectedSchedule = stringAt(spec, "schedule");
|
||||
const expectedSchedule = desiredSchedule ?? stringAt(spec, "schedule");
|
||||
const script = [
|
||||
"set +e",
|
||||
`namespace=${shellQuote(namespace)}`,
|
||||
|
||||
@@ -128,6 +128,48 @@ export function runSentinelMaintenance(state: SentinelCicdState, options: Extrac
|
||||
|
||||
export function runSentinelValidate(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "validate" }>): RenderedCliResult {
|
||||
const command = "web-probe sentinel validate";
|
||||
if (options.localOnly) {
|
||||
const cronJob = state.manifests.find((item) => item.kind === "CronJob") ?? null;
|
||||
const scenarioId = stringAt(state.cicd, "targetValidation.scenarioId");
|
||||
const scenarioRows = Array.isArray(state.scenarios)
|
||||
? state.scenarios
|
||||
: Array.isArray(record(state.scenarios).scenarios)
|
||||
? record(state.scenarios).scenarios as unknown[]
|
||||
: [];
|
||||
const scenario = scenarioRows.map(record).find((item) => item.id === scenarioId) ?? null;
|
||||
const cadence = scenario === null ? null : scenario.cadence ?? null;
|
||||
const renderedCron = cronJob === null ? null : record(cronJob.spec).schedule ?? null;
|
||||
const result = {
|
||||
ok: state.configReady && state.sourceHead.ok && cadence !== null && renderedCron !== null,
|
||||
command,
|
||||
mode: "local",
|
||||
mutation: false,
|
||||
node: state.spec.nodeId,
|
||||
lane: state.spec.lane,
|
||||
sentinelId: state.sentinelId,
|
||||
configRef: state.configRefs.scenarios,
|
||||
scenarioId,
|
||||
cadence,
|
||||
renderedCron,
|
||||
sourceCommit: state.sourceHead.commit,
|
||||
manifestSha256: state.manifestSha256,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return rendered(result.ok, command, [
|
||||
command,
|
||||
`status: ${result.ok ? "ok" : "blocked"}`,
|
||||
`node: ${result.node}`,
|
||||
`lane: ${result.lane}`,
|
||||
`sentinelId: ${result.sentinelId}`,
|
||||
`configRef: ${result.configRef}`,
|
||||
`scenarioId: ${result.scenarioId}`,
|
||||
`cadence: ${result.cadence ?? "-"}`,
|
||||
`renderedCron: ${result.renderedCron ?? "-"}`,
|
||||
`sourceCommit: ${result.sourceCommit ?? "-"}`,
|
||||
`manifestSha256: ${result.manifestSha256}`,
|
||||
"mutation: false",
|
||||
].join("\n"), result);
|
||||
}
|
||||
const startedAt = Date.now();
|
||||
const initialHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds);
|
||||
let quickVerify: Record<string, unknown> | null = null;
|
||||
|
||||
@@ -87,7 +87,7 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
|
||||
"--source-stage-ref",
|
||||
"--source-mirror-commit",
|
||||
"--source-authority",
|
||||
]), new Set(["--dry-run", "--confirm", "--wait", "--rerun", "--manual-recovery", "--recovery", "--quick-verify", "--raw", "--full", "--latest", "--full-page", "--no-full-page", "--diagnose-monitor-web"]));
|
||||
]), new Set(["--dry-run", "--confirm", "--wait", "--local", "--rerun", "--manual-recovery", "--recovery", "--quick-verify", "--raw", "--full", "--latest", "--full-page", "--no-full-page", "--diagnose-monitor-web"]));
|
||||
const nodeOption = optionValue(args, "--node") ?? null;
|
||||
const laneOption = optionValue(args, "--lane") ?? null;
|
||||
const inspectMode = sentinelActionRaw === "inspect-url" || sentinelActionRaw === "inspect-id";
|
||||
@@ -183,7 +183,7 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
|
||||
quickVerify: maintenanceAction === "stop" || args.includes("--quick-verify"),
|
||||
};
|
||||
} else if (sentinelActionRaw === "validate") {
|
||||
sentinel = { kind: "validate", action: "validate", node, lane, sentinelId, dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds, quickVerify: args.includes("--quick-verify") };
|
||||
sentinel = { kind: "validate", action: "validate", node, lane, sentinelId, dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds, quickVerify: args.includes("--quick-verify"), localOnly: args.includes("--local") };
|
||||
} else if (sentinelActionRaw === "dashboard") {
|
||||
const dashboardAction = parseWebProbeSentinelDashboardAction(args[1]);
|
||||
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 120000);
|
||||
|
||||
@@ -16,6 +16,7 @@ export interface RenderedCliResult {
|
||||
command: string;
|
||||
renderedText: string;
|
||||
contentType: "text/plain" | "application/json" | "application/yaml";
|
||||
projection?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface CliInputErrorOptions {
|
||||
|
||||
@@ -97,6 +97,7 @@ export function compactDiagnoseCodeAgentResult(value: unknown): Record<string, u
|
||||
hwlabReadModel: source.hwlabReadModel ?? null,
|
||||
http: compactDiagnoseHttp(source.http),
|
||||
projectionLag: source.projectionLag ?? null,
|
||||
outcome: source.outcome ?? null,
|
||||
summary: source.summary ?? null,
|
||||
rootCauseCandidates: asArray(source.rootCauseCandidates).slice(0, 3).map(compactRootCauseCandidate),
|
||||
spanNameCounts: compactNameCounts(source.spanNameCounts, 6),
|
||||
|
||||
@@ -123,16 +123,33 @@ describe("diagnose-code-agent warm-session terminal classification", () => {
|
||||
expect(first.payload.diagnosticTrace.traceId).not.toBe(second.payload.diagnosticTrace.traceId);
|
||||
expect(first.payload.diagnosticTrace.traceparent).not.toBe(second.payload.diagnosticTrace.traceparent);
|
||||
});
|
||||
|
||||
test("keeps durable completion authoritative while projection lag stays a nonblocking warning", () => {
|
||||
const diagnosis = runFixture({ authorityAvailable: true, terminalSpan: false, projectionStale: true });
|
||||
expect(diagnosis.status).toBe(0);
|
||||
expect(diagnosis.payload.outcome).toEqual({
|
||||
typed: "completed-with-projection-warning",
|
||||
effectiveStatus: "completed",
|
||||
authority: { kind: "agentrun-durable-command", status: "completed", source: "agentrun-authority" },
|
||||
projection: { kind: "hwlab-read-model", status: "running", lag: "confirmed", warning: true, blocking: false },
|
||||
mutation: false,
|
||||
});
|
||||
const compact = compactDiagnoseCodeAgentResult(diagnosis.payload);
|
||||
const rendered = renderDiagnoseCodeAgentTable({ ok: true, target, options, query: {}, result: compact });
|
||||
expect(rendered.renderedText).toContain("outcome=completed-with-projection-warning effectiveStatus=completed mutation=false");
|
||||
expect(rendered.renderedText).toContain("durableAuthority kind=agentrun-durable-command status=completed");
|
||||
expect(rendered.renderedText).toContain("projection kind=hwlab-read-model status=running lag=confirmed warning=true blocking=false");
|
||||
});
|
||||
});
|
||||
|
||||
function runFixture(input: { authorityAvailable: boolean; terminalSpan: boolean }): {
|
||||
function runFixture(input: { authorityAvailable: boolean; terminalSpan: boolean; projectionStale?: boolean }): {
|
||||
status: number | null;
|
||||
payload: Record<string, any>;
|
||||
calls: string;
|
||||
} {
|
||||
const directory = mkdtempSync(join(tmpdir(), "unidesk-otel-diagnose-"));
|
||||
temporaryDirectories.push(directory);
|
||||
writeFileSync(join(directory, "trace.json"), JSON.stringify(traceFixture(input.terminalSpan)));
|
||||
writeFileSync(join(directory, "trace.json"), JSON.stringify(traceFixture(input.terminalSpan, input.projectionStale === true)));
|
||||
writeFileSync(join(directory, "command.json"), JSON.stringify({ id: "cmd_second", state: "completed" }));
|
||||
writeFileSync(join(directory, "result.json"), JSON.stringify({ ok: true, commandId: "cmd_second", status: "completed", commandState: "completed", terminalStatus: "completed" }));
|
||||
writeFileSync(join(directory, "run-result.json"), JSON.stringify({ ok: true, commandId: "cmd_second", commandState: "completed", terminalStatus: "completed" }));
|
||||
@@ -235,7 +252,7 @@ function callLines(calls: string): string[] {
|
||||
return calls.split("\n").map((line) => line.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
function traceFixture(terminalSpan: boolean): Record<string, unknown> {
|
||||
function traceFixture(terminalSpan: boolean, projectionStale = false): Record<string, unknown> {
|
||||
const common = [
|
||||
attribute("traceId", "trc_warm_second"),
|
||||
attribute("runId", "run_warm"),
|
||||
@@ -244,11 +261,17 @@ function traceFixture(terminalSpan: boolean): Record<string, unknown> {
|
||||
];
|
||||
return {
|
||||
batches: [
|
||||
batch("hwlab-cloud-api", [span("provider_decision", [
|
||||
...common,
|
||||
attribute("agent.chat.provider_profile", "gpt.pika"),
|
||||
attribute("agentRunRunnerNamespace", "agentrun-v02"),
|
||||
])]),
|
||||
batch("hwlab-cloud-api", [
|
||||
span("provider_decision", [
|
||||
...common,
|
||||
attribute("agent.chat.provider_profile", "gpt.pika"),
|
||||
attribute("agentRunRunnerNamespace", "agentrun-v02"),
|
||||
]),
|
||||
...(projectionStale ? [
|
||||
span("trace_events_read", [...common, attribute("sourceEventCount", "6"), attribute("sinceSeq", "7")]),
|
||||
span("turn_status_read", [...common, attribute("turnStatus", "running")]),
|
||||
] : []),
|
||||
]),
|
||||
batch("agentrun-manager", [
|
||||
span("command_created", common),
|
||||
...(terminalSpan ? [span("runner_terminal.completed", [...common, attribute("terminalStatus", "completed"), attribute("eventType", "terminal_status")])] : []),
|
||||
|
||||
@@ -2023,10 +2023,10 @@ def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error
|
||||
status_conflict = terminal_status_conflicts(agentrun.get("terminalStatus"), read_model_turn_status)
|
||||
if status_conflict:
|
||||
candidates.append({
|
||||
"code": "otel_business_trace_terminal_conflict",
|
||||
"label": "OTel terminal conflict",
|
||||
"code": "projection_authority_terminal_conflict",
|
||||
"label": "projection/authority terminal conflict",
|
||||
"confidence": 0.86,
|
||||
"summary": "Scoped AgentRun terminal status conflicts with HWLAB read-model turn status for the requested business trace; prefer the HWLAB read-model status for Workbench RCA and inspect OTel correlation leakage.",
|
||||
"summary": "HWLAB projection conflicts with the scoped AgentRun durable authority; preserve the AgentRun terminal outcome and inspect projection correlation or lag.",
|
||||
"evidence": {
|
||||
"agentrunTerminalStatus": agentrun.get("terminalStatus"),
|
||||
"hwlabReadModelTurnStatus": read_model_turn_status,
|
||||
@@ -2502,16 +2502,16 @@ if http_summary.get("actorForbidden"):
|
||||
read_model_turn_status = read_model.get("turnStatus")
|
||||
agentrun_read_model_status_conflict = terminal_status_conflicts(terminal_status, read_model_turn_status)
|
||||
if agentrun_read_model_status_conflict:
|
||||
facts.append("OTel AgentRun terminal status conflicts with HWLAB read model")
|
||||
if terminal_status in ("failed", "error", "timeout", "blocked", "cancelled") and not agentrun_read_model_status_conflict:
|
||||
facts.append("HWLAB projection conflicts with AgentRun durable authority")
|
||||
if terminal_status in ("failed", "error", "timeout", "blocked", "cancelled"):
|
||||
failure_kind = agentrun.get("failureKind")
|
||||
if failure_kind:
|
||||
facts.append(f"AgentRun terminal failed ({failure_kind})")
|
||||
else:
|
||||
facts.append("AgentRun terminal failed")
|
||||
if terminal_status == "completed" and not agentrun_read_model_status_conflict:
|
||||
if terminal_status == "completed":
|
||||
facts.append("AgentRun completed")
|
||||
if (terminal_status in (None, "") or agentrun_read_model_status_conflict) and read_model_turn_status in ("completed", "failed", "error", "timeout", "blocked", "cancelled", "canceled"):
|
||||
if terminal_status in (None, "") and read_model_turn_status in ("completed", "failed", "error", "timeout", "blocked", "cancelled", "canceled"):
|
||||
facts.append("HWLAB read model terminal " + str(read_model_turn_status))
|
||||
if lag.get("status") in ("confirmed", "suspected"):
|
||||
facts.append("projection/read-model stale")
|
||||
@@ -2552,6 +2552,30 @@ evidence = {
|
||||
"idleWarningSpanCount": len(idle_warning_spans),
|
||||
"idleWarningSpanTail": [tiny_span(item) for item in idle_warning_spans[-3:]],
|
||||
}
|
||||
projection_warning = lag.get("status") in ("confirmed", "suspected") or agentrun_read_model_status_conflict
|
||||
effective_status = terminal_status if terminal_status not in (None, "") else read_model_turn_status
|
||||
typed_outcome = "unknown"
|
||||
if effective_status not in (None, ""):
|
||||
typed_outcome = str(effective_status)
|
||||
if projection_warning:
|
||||
typed_outcome = typed_outcome + "-with-projection-warning"
|
||||
outcome = {
|
||||
"typed": typed_outcome,
|
||||
"effectiveStatus": effective_status,
|
||||
"authority": {
|
||||
"kind": "agentrun-durable-command",
|
||||
"status": terminal_status,
|
||||
"source": agentrun.get("terminalSource"),
|
||||
},
|
||||
"projection": {
|
||||
"kind": "hwlab-read-model",
|
||||
"status": read_model_turn_status,
|
||||
"lag": lag.get("status"),
|
||||
"warning": projection_warning,
|
||||
"blocking": False,
|
||||
},
|
||||
"mutation": False,
|
||||
}
|
||||
payload = {
|
||||
"ok": trace_rc == 0 and len(spans) > 0,
|
||||
"mapping": mapping,
|
||||
@@ -2594,6 +2618,7 @@ payload = {
|
||||
"actorForbidden": http_summary.get("actorForbidden"),
|
||||
},
|
||||
"projectionLag": lag,
|
||||
"outcome": outcome,
|
||||
"summary": summary,
|
||||
"rootCauseCandidates": candidates,
|
||||
"spanNameCounts": flat["spanNameCounts"][:12],
|
||||
|
||||
@@ -761,6 +761,9 @@ export function renderDiagnoseCodeAgentTable(input: {
|
||||
const observedCommandId = textValue(identity?.commandId);
|
||||
const observedSessionId = textValue(identity?.sessionId);
|
||||
const observedRunnerJobId = textValue(identity?.runnerJobId);
|
||||
const outcome = asPlainRecord(input.result.outcome);
|
||||
const outcomeAuthority = asPlainRecord(outcome?.authority);
|
||||
const outcomeProjection = asPlainRecord(outcome?.projection);
|
||||
const lines = [
|
||||
`platform-infra observability diagnose-code-agent (${input.ok ? "ok" : "not-ok"})`,
|
||||
"",
|
||||
@@ -792,6 +795,9 @@ export function renderDiagnoseCodeAgentTable(input: {
|
||||
formatTable(["METHOD", "ROUTE", "STATUS", "COUNT"], httpRows.length > 0 ? httpRows : [["-", "-", "-", "-"]]),
|
||||
"",
|
||||
"Summary:",
|
||||
` outcome=${textValue(outcome?.typed)} effectiveStatus=${textValue(outcome?.effectiveStatus)} mutation=${textValue(outcome?.mutation)}`,
|
||||
` durableAuthority kind=${textValue(outcomeAuthority?.kind)} status=${textValue(outcomeAuthority?.status)} source=${textValue(outcomeAuthority?.source)}`,
|
||||
` projection kind=${textValue(outcomeProjection?.kind)} status=${textValue(outcomeProjection?.status)} lag=${textValue(outcomeProjection?.lag)} warning=${textValue(outcomeProjection?.warning)} blocking=${textValue(outcomeProjection?.blocking)}`,
|
||||
` target=${input.target.id} spanCount=${textValue(input.result.spanCount)} servicePath=${joinValues(input.result.servicePath, 60)}`,
|
||||
` agentrun=${textValue(agentrun?.terminalStatus)} failureKind=${textValue(agentrun?.failureKind)} source=${textValue(agentrun?.terminalSource)} authority=${shortenEnd(JSON.stringify((asPlainRecord(agentrun?.authority) ?? {})), 120)}`,
|
||||
` authorityQuery namespace=${textValue(agentrunAuthority?.namespace)} source=${textValue(agentrunAuthority?.namespaceSource)} ok=${textValue(agentrunAuthority?.ok)} commandState=${textValue(agentrunAuthority?.commandState)} terminal=${textValue(agentrunAuthority?.terminalStatus)}`,
|
||||
|
||||
@@ -86,10 +86,11 @@ describe("ssh bounded progressive help", () => {
|
||||
test("renders compact top-level and scoped help without dump fallback", () => {
|
||||
const top = runTransHelp("--help");
|
||||
expect(Buffer.byteLength(top, "utf8")).toBeLessThan(2_048);
|
||||
expect(top.trim().split("\n")).toHaveLength(10);
|
||||
expect(top.trim().split("\n")).toHaveLength(11);
|
||||
expect(top).toContain("usage: trans <route> <operation>");
|
||||
expect(top).toContain("transfer: upload | download");
|
||||
expect(top).toContain("scoped help: trans --help <operation>");
|
||||
expect(top).toContain("cwd: host/workspace uses <provider>:/absolute/workspace; k3s exec uses --cwd");
|
||||
expect(top).not.toContain("outputTruncated");
|
||||
expect(top).not.toContain("dump");
|
||||
|
||||
@@ -102,6 +103,10 @@ describe("ssh bounded progressive help", () => {
|
||||
expect(applyPatch).toContain("usage: trans <route> apply-patch");
|
||||
expect(applyPatch).not.toContain("outputTruncated");
|
||||
|
||||
const exec = runTransHelp("--help", "exec");
|
||||
expect(exec).toContain("trans <provider>:/absolute/workspace exec <command>");
|
||||
expect(exec).toContain("k3s:<namespace>:<workload>");
|
||||
|
||||
const download = runTransHelp("--help", "download");
|
||||
expect(download.trim().split("\n").length).toBeLessThanOrEqual(8);
|
||||
expect(download).toContain("TRANS HELP download");
|
||||
@@ -159,4 +164,17 @@ describe("ssh bounded progressive help", () => {
|
||||
expect(result.stdout).not.toContain("stack");
|
||||
expect(result.stdout).not.toContain(" at ");
|
||||
});
|
||||
|
||||
test("rejects the moved root entrypoint locally with one exact replacement", () => {
|
||||
const result = spawnSync("bun", ["scripts/cli.ts", "trans", "NC01:/root/unidesk", "git", "status", "--short"], {
|
||||
cwd: repoRoot,
|
||||
env: { ...process.env, UNIDESK_TRANS_REPO_ROOT: repoRoot },
|
||||
encoding: "utf8",
|
||||
});
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toBe("");
|
||||
expect(result.stdout).toContain("ERROR cli-input/trans-root-cli-entrypoint-moved");
|
||||
expect(result.stdout).toContain("usage: trans NC01:/root/unidesk git status --short");
|
||||
expect(result.stdout).not.toContain("stack");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1372,11 +1372,12 @@ function writeChunkedStdin(stdin: NodeJS.WritableStream, input: string): void {
|
||||
export async function runSsh(config: UniDeskConfig, providerId: string, args: string[]): Promise<number> {
|
||||
const normalizedArgs = normalizeSshOperationArgs(args);
|
||||
process.stderr.write(sshRouteSeparatorCompatibilityHint(args, normalizedArgs));
|
||||
// 在选择或连接 transport 前完成解析,保证可预测的 route/operation 错误始终在本地失败。
|
||||
const invocation = parseSshInvocation(providerId, normalizedArgs);
|
||||
const plan = sshCaptureBackendPlan(config, process.env);
|
||||
if (plan.backend === "remote-frontend-websocket" && plan.remoteHost !== null) {
|
||||
return await runRemoteSsh(config, plan.remoteHost, providerId, normalizedArgs);
|
||||
}
|
||||
const invocation = parseSshInvocation(providerId, normalizedArgs);
|
||||
const parsed = invocation.parsed;
|
||||
const operationName = normalizedArgs[0] ?? "";
|
||||
if (isSshFileTransferOperation(normalizedArgs)) {
|
||||
|
||||
@@ -214,6 +214,20 @@ describe("ssh windows fs read-only operations", () => {
|
||||
});
|
||||
|
||||
describe("ssh direct argv boundaries and plane diagnostics", () => {
|
||||
test("rejects host/workspace cwd options locally with an exact route replacement", () => {
|
||||
try {
|
||||
parseSshInvocation("NC01:/root/unidesk", ["exec", "--cwd", "/root/hwlab-v03", "--", "git", "status", "--short"]);
|
||||
throw new Error("expected host cwd input error");
|
||||
} catch (error) {
|
||||
expect(error).toMatchObject({
|
||||
code: "trans-host-cwd-belongs-in-route",
|
||||
usage: "trans NC01:/root/hwlab-v03 'exec' '--' 'git' 'status' '--short'",
|
||||
});
|
||||
}
|
||||
expect(() => parseSshInvocation("NC01:/root/unidesk", ["bash", "--cwd=/root/hwlab-v03"])).toThrow("cwd belongs in the route");
|
||||
expect(() => parseSshInvocation("NC01:k3s:hwlab-v03:hwlab-cloud-api", ["exec", "--cwd", "/workspace", "--", "pwd"])).not.toThrow();
|
||||
});
|
||||
|
||||
test("preserves a shell-formed argv token containing spaces and Chinese", () => {
|
||||
const invocation = parseSshInvocation("G14-WSL:/mnt/d/Work/CONSTAR_workspace", [
|
||||
"python3",
|
||||
|
||||
@@ -3,6 +3,7 @@ import { isSshFileTransferOperation } from "./ssh-file-transfer";
|
||||
import { isWindowsFsReadOnlyOperation, windowsFsReadOnlyScript } from "./ssh-windows-fs";
|
||||
import { remoteApplyPatchSource, remoteGlobSource, remoteSkillDiscoverSource } from "./ssh-remote-tools";
|
||||
import { readTransHostProxyEnvRule, type TransHostProxyEnvRule } from "./trans-host-proxy";
|
||||
import { CliInputError } from "./output";
|
||||
|
||||
export interface ParsedSshArgs {
|
||||
remoteCommand: string | null;
|
||||
@@ -406,10 +407,31 @@ export function parseSshInvocation(target: string, args: string[]): ParsedSshInv
|
||||
if ((operationArgs[0] ?? "") === "k3s") {
|
||||
throw new Error(`ssh k3s shorthand is unsupported; use route syntax instead: trans ${route.providerId}:k3s ${operationArgs.slice(1).join(" ")}`.trim());
|
||||
}
|
||||
rejectHostWorkspaceCwdOption(route, operationArgs);
|
||||
const parsed = parseSshArgs(operationArgs, route.raw);
|
||||
return { providerId: route.providerId, route, parsed: withTransHostEnvironment(route, parsed) };
|
||||
}
|
||||
|
||||
function rejectHostWorkspaceCwdOption(route: ParsedSshRoute, args: string[]): void {
|
||||
const operation = args[0] ?? "";
|
||||
if (!new Set(["exec", "argv", "sh", "bash"]).has(operation)) return;
|
||||
const cwdIndex = args.findIndex((arg) => arg === "--cwd" || arg === "--workdir" || arg.startsWith("--cwd=") || arg.startsWith("--workdir="));
|
||||
if (cwdIndex < 0) return;
|
||||
const option = args[cwdIndex] ?? "--cwd";
|
||||
const inlineValue = option.includes("=") ? option.slice(option.indexOf("=") + 1) : null;
|
||||
const cwd = inlineValue ?? args[cwdIndex + 1] ?? "<absolute-workspace>";
|
||||
const consumed = inlineValue === null ? 2 : 1;
|
||||
const remaining = [...args.slice(0, cwdIndex), ...args.slice(cwdIndex + consumed)];
|
||||
const replacementRoute = cwd.startsWith("/") ? `${route.providerId}:${cwd}` : `${route.providerId}:/absolute/workspace`;
|
||||
const replacement = `trans ${replacementRoute} ${remaining.map(shellQuote).join(" ")}`;
|
||||
throw new CliInputError(`host/workspace ${operation} does not accept ${option.split("=", 1)[0]}; cwd belongs in the route`, {
|
||||
code: "trans-host-cwd-belongs-in-route",
|
||||
argument: option,
|
||||
usage: replacement,
|
||||
hint: "Use <provider>:/absolute/workspace for host cwd. Use exec --cwd only with a k3s workload route.",
|
||||
});
|
||||
}
|
||||
|
||||
export function parseSshRoute(target: string): ParsedSshRoute {
|
||||
if (!target) throw new Error("ssh requires provider id, for example: trans D601");
|
||||
const firstColon = target.indexOf(":");
|
||||
|
||||
@@ -12,7 +12,12 @@ export function webProbeHelp(): Record<string, unknown> {
|
||||
|
||||
export async function runWebProbeCommand(_config: Config, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
if (args.length === 0 || args.includes("--help") || args.includes("-h") || args[0] === "help") return webProbeHelp();
|
||||
return runNodeWebProbe(parseNodeWebProbeOptions(args));
|
||||
const format = args.includes("--json") ? "json" : args.includes("--yaml") ? "yaml" : "text";
|
||||
const parsedArgs = args.filter((item) => item !== "--json" && item !== "--yaml");
|
||||
const result = runNodeWebProbe(parseNodeWebProbeOptions(parsedArgs));
|
||||
if (format === "text" || !("projection" in result) || result.projection === undefined) return result;
|
||||
if (format === "json") return result.projection;
|
||||
return { ...result, renderedText: Bun.YAML.stringify(result.projection), contentType: "application/yaml" };
|
||||
}
|
||||
|
||||
export function legacyHwlabNodeWebProbeUnsupported(args: string[]): Record<string, unknown> {
|
||||
|
||||
Reference in New Issue
Block a user