fix(cicd): bound node trigger wait output (#646)
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
@@ -30,7 +30,8 @@ export function hwlabNodeHelp(): Record<string, unknown> {
|
||||
"bun scripts/cli.ts hwlab nodes control-plane public-exposure --node D601 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane public-exposure --node D601 --lane v03 --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node D601 --lane v03 --confirm --rerun",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node D601 --lane v03 --confirm --wait",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node D601 --lane v03 --confirm --wait --rerun",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node D601 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node D601 --lane v03 --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane allow-endpoint-bridge --node G14 --lane v03 --confirm",
|
||||
@@ -58,6 +59,8 @@ export function hwlabNodeHelp(): Record<string, unknown> {
|
||||
],
|
||||
notes: [
|
||||
"`trigger-current` skips an already succeeded PipelineRun for the same HWLAB source commit by default.",
|
||||
"`trigger-current --confirm --wait` is the one-command CICD path: git-mirror pre-sync/pre-flush, control-plane refresh, PipelineRun create/reuse, bounded wait, and post-flush when terminal.",
|
||||
"`--wait` defaults to 120 seconds. If the PipelineRun is still active after 120 seconds, the CLI returns a warning plus env-reuse and git-mirror inspection commands instead of blocking.",
|
||||
"Use `--rerun` for a deliberate YAML-first config-only publish when UniDesk node/lane render inputs changed but the HWLAB source commit did not."
|
||||
],
|
||||
};
|
||||
|
||||
+134
-16
@@ -289,6 +289,7 @@ const CODE_AGENT_PROVIDER_OPENCODE_KEY = "opencode-api-key";
|
||||
const CODE_AGENT_PROVIDER_SOURCE_NAMESPACE = "hwlab-v02";
|
||||
const CODE_AGENT_PROVIDER_SOURCE_SECRET = "hwlab-v02-code-agent-provider";
|
||||
const HWLAB_CI_NAMESPACE = "hwlab-ci";
|
||||
const NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS = 120;
|
||||
|
||||
export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
if (args.length === 0) return hwlabNodeHelp();
|
||||
@@ -352,7 +353,7 @@ function parseNodeObservabilityOptions(args: string[]): NodeObservabilityOptions
|
||||
};
|
||||
}
|
||||
|
||||
async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise<Record<string, unknown>> {
|
||||
async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
const scoped = parseNodeScopedDelegatedOptions(domain, args);
|
||||
const defaultSpec = hwlabRuntimeLaneSpec(scoped.lane);
|
||||
if (domain === "control-plane" && scoped.action === "public-exposure") {
|
||||
@@ -439,7 +440,7 @@ function parseNodeScopedDelegatedOptions(domain: DelegatedNodeDomain, args: stri
|
||||
wait: args.includes("--wait"),
|
||||
rerun: args.includes("--rerun"),
|
||||
allowLiveDbRead: args.includes("--allow-live-db-read"),
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 1800, 3600),
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS, 3600),
|
||||
originalArgs: [...args],
|
||||
spec,
|
||||
};
|
||||
@@ -1825,6 +1826,21 @@ function shortSha(value: string): string {
|
||||
return value.slice(0, 12).toLowerCase();
|
||||
}
|
||||
|
||||
function shortValue(value: unknown): string {
|
||||
if (typeof value !== "string" || value.length === 0) return "-";
|
||||
if (/^[0-9a-f]{40}$/iu.test(value)) return shortSha(value);
|
||||
return value.length > 30 ? `${value.slice(0, 27)}~` : value;
|
||||
}
|
||||
|
||||
function formatElapsedMs(value: unknown): string {
|
||||
const ms = Number(value);
|
||||
if (!Number.isFinite(ms) || ms < 0) return "-";
|
||||
const seconds = Math.round(ms / 1000);
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const rest = seconds % 60;
|
||||
return minutes > 0 ? `${minutes}m${String(rest).padStart(2, "0")}s` : `${seconds}s`;
|
||||
}
|
||||
|
||||
function nodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string {
|
||||
return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`;
|
||||
}
|
||||
@@ -1891,11 +1907,11 @@ function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string {
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function nodeRuntimeControlPlaneRun(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
||||
function nodeRuntimeControlPlaneRun(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> | RenderedCliResult {
|
||||
if (scoped.action === "refresh") return nodeRuntimeRefresh(scoped);
|
||||
if (scoped.action === "sync") return nodeRuntimeSync(scoped);
|
||||
if (scoped.action === "apply") return nodeRuntimeApply(scoped);
|
||||
if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrent(scoped);
|
||||
if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrentOutput(scoped);
|
||||
if (scoped.action === "runtime-migration") return nodeRuntimeMigration(scoped);
|
||||
if (scoped.action === "cleanup-runs") return nodeRuntimeCleanupRuns(scoped);
|
||||
return nodeRuntimeUnsupportedAction(scoped);
|
||||
@@ -2635,8 +2651,15 @@ function nodeRuntimeSync(scoped: ReturnType<typeof parseNodeScopedDelegatedOptio
|
||||
};
|
||||
}
|
||||
|
||||
function nodeRuntimeTriggerCurrentOutput(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> | RenderedCliResult {
|
||||
const result = nodeRuntimeTriggerCurrent(scoped);
|
||||
if (nodeScopedFullOutput(scoped)) return result;
|
||||
return withNodeRuntimeTriggerRendered(result, scoped);
|
||||
}
|
||||
|
||||
function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
||||
const spec = scoped.spec;
|
||||
const pipelineWaitSeconds = nodeRuntimeCicdWaitSeconds(scoped);
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "started" });
|
||||
const head = resolveNodeRuntimeLaneHead(spec);
|
||||
const sourceCommit = head.sourceCommit;
|
||||
@@ -2678,7 +2701,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
|
||||
}
|
||||
if (!scoped.rerun && before.exists === true && (before.status === "True" || before.status === "Unknown")) {
|
||||
const pipelineWait = before.status === "Unknown"
|
||||
? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, scoped.timeoutSeconds, {
|
||||
? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, pipelineWaitSeconds, {
|
||||
opportunisticPostSync: () => nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun),
|
||||
opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun),
|
||||
})
|
||||
@@ -2688,6 +2711,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
|
||||
const postFlush = waitedPipelineRun.status === "True"
|
||||
? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun)
|
||||
: null;
|
||||
const pipelinePending = pipelineWait.status === "pending";
|
||||
const ok = pipelineWait.ok === true && (postFlush === null || postFlush.ok === true);
|
||||
return {
|
||||
ok,
|
||||
@@ -2695,6 +2719,8 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
mode: "confirmed-trigger",
|
||||
completion: pipelinePending ? "pending" : ok ? "completed" : "failed",
|
||||
warning: pipelinePending ? pipelineWait.warning ?? nodeRuntimeCicdWaitWarning(spec, pipelineRun, pipelineWait) : undefined,
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
before,
|
||||
@@ -2770,7 +2796,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
|
||||
const createOk = isCommandSuccess(create) || createObserved;
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: createOk ? "succeeded" : "failed", sourceCommit, pipelineRun, exitCode: create.exitCode, createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined });
|
||||
const pipelineWait = createOk
|
||||
? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, scoped.timeoutSeconds, {
|
||||
? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, pipelineWaitSeconds, {
|
||||
opportunisticPostSync: () => nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun),
|
||||
opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun),
|
||||
})
|
||||
@@ -2782,13 +2808,16 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
|
||||
: null;
|
||||
const pipelineReady = pipelineWait !== null && pipelineWait.ok === true;
|
||||
const postFlushOk = postFlush === null || postFlush.ok === true;
|
||||
const ok = createOk && pipelineReady && postFlushOk;
|
||||
const pipelinePending = pipelineWait !== null && pipelineWait.status === "pending";
|
||||
const ok = createOk && (pipelineReady || pipelinePending) && postFlushOk;
|
||||
return {
|
||||
ok,
|
||||
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
mode: "confirmed-trigger",
|
||||
completion: pipelinePending ? "pending" : ok ? "completed" : "failed",
|
||||
warning: pipelinePending ? pipelineWait?.warning ?? nodeRuntimeCicdWaitWarning(spec, pipelineRun, pipelineWait) : undefined,
|
||||
mutation: createOk,
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
@@ -2814,6 +2843,95 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
|
||||
};
|
||||
}
|
||||
|
||||
function nodeRuntimeCicdWaitSeconds(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): number {
|
||||
return Math.min(scoped.timeoutSeconds, NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS);
|
||||
}
|
||||
|
||||
function nodeRuntimeCicdWaitWarning(spec: HwlabRuntimeLaneSpec, pipelineRun: string, pipelineWait: unknown): Record<string, unknown> {
|
||||
const wait = record(pipelineWait);
|
||||
return {
|
||||
code: "node-runtime-cicd-wait-over-120s",
|
||||
message: `PipelineRun ${pipelineRun} is still active after ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s; CICD command stopped waiting and left the deployment running.`,
|
||||
waitedSeconds: NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS,
|
||||
elapsedMs: wait.elapsedMs ?? null,
|
||||
polls: wait.polls ?? null,
|
||||
inspectEnvReuse: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
|
||||
inspectGitMirror: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
|
||||
};
|
||||
}
|
||||
|
||||
function withNodeRuntimeTriggerRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
|
||||
const pipelineWait = record(result.pipelineWait);
|
||||
const pipelineRunRecord = record(pipelineWait.pipelineRun ?? result.after ?? result.before);
|
||||
const warning = record(result.warning ?? pipelineWait.warning);
|
||||
const postFlush = record(result.postFlush);
|
||||
const gitMirror = record(result.gitMirror);
|
||||
const gitMirrorSummary = record(gitMirror.statusSummary ?? gitMirror.afterSummary ?? gitMirror.beforeSummary ?? gitMirror.summary);
|
||||
const refresh = record(result.refresh);
|
||||
const create = record(result.create);
|
||||
const next = record(result.next);
|
||||
const pipelineStatus = pipelineRunRecord.status ?? pipelineWait.status ?? "-";
|
||||
const completion = result.completion ?? (result.ok === true ? "completed" : "failed");
|
||||
const renderedText = [
|
||||
"hwlab nodes control-plane trigger-current",
|
||||
"",
|
||||
webObserveTable(
|
||||
["NODE", "LANE", "STATUS", "COMPLETION", "SOURCE", "PIPELINERUN"],
|
||||
[[scoped.node, scoped.lane, result.ok === true ? "ok" : "failed", webObserveText(completion), shortValue(result.sourceCommit), webObserveText(result.pipelineRun)]],
|
||||
),
|
||||
"",
|
||||
webObserveTable(
|
||||
["STAGE", "STATUS", "DETAIL"],
|
||||
[
|
||||
["git-mirror", gitMirror.ok === true ? "ok" : gitMirror.ok === false ? "failed" : "-", webObserveText(gitMirror.mode ?? gitMirror.degradedReason)],
|
||||
["refresh", refresh.ok === true ? "ok" : refresh.ok === false ? "failed" : "-", webObserveText(refresh.degradedReason)],
|
||||
["create", create.exitCode === 0 || result.mutation === true ? "ok" : create.exitCode === undefined ? "-" : "failed", result.createObservedAfterTimeout === true ? "observed-after-timeout" : `exit=${webObserveText(create.exitCode)}`],
|
||||
["pipeline-wait", webObserveText(pipelineWait.status), `pipeline=${webObserveText(pipelineStatus)} polls=${webObserveText(pipelineWait.polls)} elapsed=${formatElapsedMs(pipelineWait.elapsedMs)}`],
|
||||
["post-flush", postFlush.ok === true ? "ok" : postFlush.ok === false ? "failed" : "-", webObserveText(postFlush.mode ?? postFlush.degradedReason)],
|
||||
],
|
||||
),
|
||||
"",
|
||||
Object.keys(warning).length === 0
|
||||
? "WARNING\n-"
|
||||
: webObserveTable(
|
||||
["CODE", "MESSAGE"],
|
||||
[[webObserveText(warning.code ?? "warning"), webObserveText(warning.message ?? "PipelineRun is still active after the default wait window.")]],
|
||||
),
|
||||
"",
|
||||
Object.keys(warning).length === 0
|
||||
? "INSPECT\n-"
|
||||
: webObserveTable(
|
||||
["CHECK", "COMMAND"],
|
||||
[
|
||||
["env-reuse", webObserveText(warning.inspectEnvReuse ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${result.pipelineRun} --full`)],
|
||||
["git-mirror", webObserveText(warning.inspectGitMirror ?? `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`)],
|
||||
],
|
||||
),
|
||||
"",
|
||||
Object.keys(gitMirrorSummary).length === 0
|
||||
? "GIT_MIRROR\n-"
|
||||
: webObserveTable(
|
||||
["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"],
|
||||
[[
|
||||
shortValue(gitMirrorSummary.localSource),
|
||||
shortValue(gitMirrorSummary.githubSource),
|
||||
shortValue(gitMirrorSummary.localGitops),
|
||||
shortValue(gitMirrorSummary.githubGitops),
|
||||
webObserveText(gitMirrorSummary.pendingFlush),
|
||||
webObserveText(gitMirrorSummary.githubInSync),
|
||||
]],
|
||||
),
|
||||
"",
|
||||
"NEXT",
|
||||
` ${next.status ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${result.pipelineRun}`}`,
|
||||
"",
|
||||
"Disclosure:",
|
||||
" default view is a bounded CICD summary; use --full or --raw for the complete JSON payload.",
|
||||
` default wait is ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s; if exceeded, inspect env-reuse and git-mirror before rerun.`,
|
||||
].join("\n");
|
||||
return { ...result, renderedText, contentType: "text/plain" };
|
||||
}
|
||||
|
||||
function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
||||
const spec = scoped.spec;
|
||||
const mirror = nodeRuntimeGitMirrorTarget(spec);
|
||||
@@ -5802,20 +5920,20 @@ function waitForNodeRuntimePipelineRunTerminal(
|
||||
}
|
||||
sleepSync(Math.min(10_000, Math.max(1000, deadline - Date.now())));
|
||||
}
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "timeout", pipelineRun, polls, elapsedMs: Date.now() - startedAt });
|
||||
const diagnostics = nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
|
||||
const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
|
||||
const elapsedMs = Date.now() - startedAt;
|
||||
const warning = nodeRuntimeCicdWaitWarning(spec, pipelineRun, { polls, elapsedMs });
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "warning", pipelineRun, polls, elapsedMs, waitLimitSeconds: timeoutSeconds });
|
||||
return {
|
||||
ok: false,
|
||||
status: "timeout",
|
||||
ok: true,
|
||||
status: "pending",
|
||||
pipelineRun: last,
|
||||
diagnostics,
|
||||
failureSummary,
|
||||
polls,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
elapsedMs,
|
||||
waitLimitSeconds: timeoutSeconds,
|
||||
warning,
|
||||
opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
|
||||
opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
|
||||
degradedReason: "node-runtime-pipelinerun-wait-timeout",
|
||||
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun}` },
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user