fix: bound hwlab g14 cli output
This commit is contained in:
@@ -43,10 +43,10 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 DEV/PROD 滚动、P
|
||||
- `artifact-registry plan|render|status|health|install|deploy-backend-core|deploy-service` 管理 D601 host-managed CNCF Distribution registry 的声明、安装、只读检查和 pull-only artifact CD。该 registry 固定为 D601 loopback `127.0.0.1:5000`,由 systemd + Docker Compose 管理,位于 native k3s 故障域外;`deploy-service` 只拉取 CI 已发布的 commit-pinned 镜像、retag/recreate 或导入 native k3s,并做 live commit 验证,不构建 runtime source。`deploy-backend-core` 是 deprecated 兼容名,标准 backend-core prod CD 入口是 `deploy apply --env prod --service backend-core`。长期规则见 `docs/reference/artifact-registry.md`。
|
||||
- `commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr` 是 host Codex 指挥官直管微服务 skeleton 入口。当前命令返回 `phase=source-contract`、service/API/state/bridge/prompt/trace/#20/#46/ClaudeQQ 审批边界、.state/commander/ 状态模型、dev 无 daemon smoke contract、dry-run 计划和 GPT-5.5 PR prompt 边界辅助 lint,不接 live bridge、不注入 prompt、不发送 ClaudeQQ。`approval request --dry-run` 会生成 200 字以内中文纯文本 ClaudeQQ 审批草案、`notification-path-unavailable` blocker 和授权后唯一可用的 `bun scripts/cli.ts microservice proxy claudeqq /api/push/text --method POST --body-json '<payload>' --raw` 命令;不得提示使用本机 ClaudeQQ skill、powershell 或本地 server。`prompt-lint` 支持 `--prompt-file` 与 `--stdin`,输出 `ok`、`missingClauses`、`riskLevel`、`suggestedPatchSnippet` 且不回显完整 prompt;它是 commander 辅助检查,不是业务 PR 门禁,也不改变 `codex submit` 默认行为。`plan`、`smoke` 与 `approval request` 必须带 `--dry-run`;缺少时返回 `error=dry-run-required`。长期规则见 `docs/reference/host-codex-commander.md`。
|
||||
- `hwlab g14 monitor-prs [--once] [--dry-run] [--interval-seconds N] [--max-cycles N] [--timeout-seconds N]` 是当前 HWLAB G14 PR -> CI/CD -> DEV rollout 的一行式入口。普通调用创建 `.state/jobs/` 异步 job 并立刻返回 `job.id`、`statusCommand` 和 stdout/stderr 路径;后台 worker 每轮通过 UniDesk `gh pr list/preflight/merge` 监控 `pikasTech/HWLAB` base=`G14` 的 open PR,ready 时合并,然后通过 UniDesk `ssh G14:k3s` 观察 `hwlab-g14-ci-poll-<short>`、Argo `hwlab-g14-dev` 和 DEV `/health/live`,直到 DEV `Synced/Healthy` 且 Deployment/StatefulSet ready;历史 `Completed` smoke/debug pod 不作为 rollout blocker。每次成功 DEV rollout 后,worker 会定位或创建 #7“指挥简报索引”中的北京日期每日简报 issue,并追加 CI/CD 耗时、CI/CD 关键指标、语义化上线 changelog、自动 diff 摘要、PipelineRun、GitOps revision 和 DEV 验证摘要;关键指标来自 G14 Tekton TaskRun results,固定包含 `lazy build reused: x/y`、reused services、rebuild services 和每个 service 的独立耗时/状态/backend,用于观察 lazy build 机制效果。语义化 changelog 优先从 PR body 的 `## 修改`/`## 变更`/`## Changelog` 等段落提取,diff 摘要只作为文件和统计证据保留,不替代 changelog。也可用 `hwlab g14 record-rollout --pr <number> --source-commit <sha>` 手动补记,手动补记同样会按 PipelineRun 采集 TaskRun 指标。状态指针按用途分离:长期监控只写 `.state/hwlab-g14/latest-monitor-job.json`,`--once` 写 `latest-once-job.json`,`--dry-run` 写 `latest-dry-run-job.json`,`--once --dry-run` 写 `latest-once-dry-run-job.json`,避免一次性收口覆盖持续监控入口。`--once --dry-run` 只做单轮监控和 merge plan,不写 GitHub、不等待 rollout。该命令禁止使用原生 `gh` 或手拼 GitHub 请求;如果 UniDesk `gh` 子命令字段或行为不够,必须先改进 `scripts/src/gh.ts` 后再使用。
|
||||
- `hwlab g14 control-plane status|apply --lane v02 [--dry-run|--confirm]` 是 HWLAB `v0.2` 加法 lane 的受控 Tekton/Argo 控制面维护入口,只面向 G14 `/root/hwlab-v02`、branch `v0.2`、namespace `hwlab-ci` 和 Argo application `hwlab-g14-v02`;`status` 只读汇总 pipeline、RBAC/ServiceAccount、Argo、当前 commit PipelineRun 和遗留 v02 CronJob 清理状态;`apply` 先在 G14 workspace 快进并执行 render check,再经 `G14:k3s` server-side apply `tekton-v02/rbac.yaml`、`pipeline.yaml`、`argocd/project.yaml` 和 `argocd/application-v02.yaml`,confirmed apply 会删除遗留 v02 CronJob,但不会应用 runtime-v02 workload、Secret 或数据迁移。
|
||||
- `hwlab g14 control-plane status|apply --lane v02 [--dry-run|--confirm]` 是 HWLAB `v0.2` 加法 lane 的受控 Tekton/Argo 控制面维护入口,只面向 G14 `/root/hwlab-v02`、branch `v0.2`、namespace `hwlab-ci` 和 Argo application `hwlab-g14-v02`;`status` 只读汇总 pipeline、RBAC/ServiceAccount、Argo、当前 commit PipelineRun、最近 PipelineRun 摘要、活跃 PipelineRun 和遗留 v02 CronJob 清理状态,默认只读取必要字段,禁止把完整 PipelineRun spec、Tekton 内联脚本或历史大对象展开到默认输出;`apply` 先在 G14 workspace 快进并执行 render check,再经 `G14:k3s` server-side apply `tekton-v02/rbac.yaml`、`pipeline.yaml`、`argocd/project.yaml` 和 `argocd/application-v02.yaml`,confirmed apply 会删除遗留 v02 CronJob,但不会应用 runtime-v02 workload、Secret 或数据迁移。
|
||||
- `hwlab g14 control-plane trigger-current --lane v02 [--dry-run|--confirm]` 是 v02 标准手动触发入口:解析当前 `origin/v0.2` full SHA,创建 commit-pinned `hwlab-v02-ci-poll-<short12>` PipelineRun;读 Git 走 `git-mirror-http.devops-infra.svc.cluster.local`,GitOps promotion 写 `git-mirror-write.devops-infra.svc.cluster.local`;confirmed trigger 在删除/创建 PipelineRun 前会先按当前 source commit render 并 server-side apply v02 Tekton RBAC、Pipeline 与 Argo Application,避免 CI/CD 脚本或 runtime-ready 逻辑已合并但集群仍执行旧 Pipeline 定义;同名 PipelineRun 成功或运行中时拒绝重复触发,失败或不存在时才删除旧对象并重新创建。
|
||||
创建 PipelineRun 前会读取 `devops-infra` mirror refs,若 `localV02` 未等于当前 source commit,则自动执行一次受控 manual `git-mirror sync` Job 并复核 ref,复核失败时停止触发,避免 Tekton `prepare-source` 已知失败;services 参数只包含 v02 runtime service matrix,`hwlab-cli` 是固定 repo 短连接源码工具,不进入 PipelineRun service build。
|
||||
`--dry-run` 只报告是否会 pre-sync,不创建 Job;confirmed trigger 默认创建 `.state/jobs/` 异步 job 并立刻返回 `job.id`、`statusCommand`、stdout/stderr 路径,避免 git mirror pre-sync 或 PipelineRun 创建期间长时间阻塞;`--wait` 路径也必须向 stderr 输出 `hwlab.v02.trigger.progress` JSON 事件,覆盖 `control-plane-refresh`、`git-mirror-pre-sync`、`delete-existing-pipelinerun` 和 `create-pipelinerun`,避免异步 job 长时间只有启动命令而无法判断卡点;只有现场同步调试才显式加 `--wait`;旧 `rerun-current` 只作为输入别名保留。
|
||||
`--dry-run` 只报告是否会 pre-sync,不创建 Job;confirmed trigger 默认创建 `.state/jobs/` 异步 job 并立刻返回 `job.id`、`statusCommand`、stdout/stderr 路径,避免 git mirror pre-sync 或 PipelineRun 创建期间长时间阻塞;`--wait` 路径也必须向 stderr 输出 `hwlab.v02.trigger.progress` JSON 事件,覆盖 `control-plane-refresh`、`git-mirror-pre-sync`、`delete-existing-pipelinerun` 和 `create-pipelinerun`,避免异步 job 长时间只有启动命令而无法判断卡点;默认 JSON 必须对 `manifest_b64`、长脚本和远端 stdout/stderr 做有界摘要,保留长度与 hash,完整内容通过 job stdout/stderr 文件渐进披露;只有现场同步调试才显式加 `--wait`;旧 `rerun-current` 只作为输入别名保留。
|
||||
- `hwlab g14 control-plane runtime-migration --lane v02 [--dry-run|--allow-live-db-read --dry-run|--confirm]` 只通过 `hwlab-v02` namespace 当前 `deployment/hwlab-cloud-api -c hwlab-cloud-api` 内 repo-owned migration CLI 执行;不读取或打印 Secret 值、不触碰 PROD、不绕到手工 `psql`。
|
||||
- `hwlab g14 control-plane cleanup-runs --lane v02|g14|all [--min-age-minutes N] [--limit N] [--dry-run|--confirm]` 是完成态 PipelineRun 工作区 retention 入口;真实清理只删除已完成 PipelineRun,让 Tekton/local-path 回收临时 PVC,不触碰 registry storage、业务 PVC、Secret、runtime workload 或 GitOps desired state。
|
||||
- `hwlab g14 control-plane cleanup-released-pvs --lane all [--limit N] [--dry-run|--confirm]` 是 local-path 未自动回收后的补充 retention 入口;只列并删除 `Released`、`local-path`、`Delete`、`claimNamespace=hwlab-ci` 且 claim 名称形如 Tekton 临时 `pvc-*` 的 PV。
|
||||
|
||||
+364
-56
@@ -1,5 +1,6 @@
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "./config";
|
||||
import { runCommand } from "./command";
|
||||
import { startJob } from "./jobs";
|
||||
@@ -114,6 +115,11 @@ interface CommandJsonResult {
|
||||
parsed: unknown | null;
|
||||
}
|
||||
|
||||
interface ShellSection {
|
||||
stdout: string;
|
||||
exitCode: number | null;
|
||||
}
|
||||
|
||||
interface OpenPullRequest {
|
||||
number: number;
|
||||
title?: string;
|
||||
@@ -344,7 +350,68 @@ function expandedParsedRoot(result: CommandJsonResult): Record<string, unknown>
|
||||
}
|
||||
|
||||
function commandData(result: CommandJsonResult): Record<string, unknown> {
|
||||
return record(expandedParsedRoot(result).data);
|
||||
return record(sanitizeCliOutput(record(expandedParsedRoot(result).data)));
|
||||
}
|
||||
|
||||
function textHash(value: string): string {
|
||||
return createHash("sha256").update(value).digest("hex").slice(0, 12);
|
||||
}
|
||||
|
||||
function redactLargePayloads(value: string): string {
|
||||
return value
|
||||
.replace(/manifest_b64='([^']{128,})'/gu, (_match, payload: string) => {
|
||||
return `manifest_b64='<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>'`;
|
||||
})
|
||||
.replace(/manifest_b64="([^"]{128,})"/gu, (_match, payload: string) => {
|
||||
return `manifest_b64="<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>"`;
|
||||
})
|
||||
.replace(/manifest_b64=([A-Za-z0-9+/=]{128,})/gu, (_match, payload: string) => {
|
||||
return `manifest_b64=<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>`;
|
||||
})
|
||||
.replace(/[A-Za-z0-9+/=]{1024,}/gu, (payload: string) => {
|
||||
return `<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>`;
|
||||
});
|
||||
}
|
||||
|
||||
function compactInlineText(value: string, maxChars = 4000): string {
|
||||
const redacted = redactLargePayloads(value);
|
||||
if (redacted.length <= maxChars) return redacted;
|
||||
const headChars = Math.floor(maxChars * 0.35);
|
||||
const tailChars = maxChars - headChars;
|
||||
const omittedChars = redacted.length - headChars - tailChars;
|
||||
return [
|
||||
redacted.slice(0, headChars),
|
||||
`<truncated chars=${omittedChars} sha256=${textHash(redacted)}>`,
|
||||
redacted.slice(-tailChars),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function sanitizeCliOutput(value: unknown): unknown {
|
||||
if (typeof value === "string") return compactInlineText(value);
|
||||
if (Array.isArray(value)) return value.map((item) => sanitizeCliOutput(item));
|
||||
if (typeof value !== "object" || value === null) return value;
|
||||
const output: Record<string, unknown> = {};
|
||||
for (const [key, item] of Object.entries(value)) output[key] = sanitizeCliOutput(item);
|
||||
return output;
|
||||
}
|
||||
|
||||
function redactedCommand(command: string[]): string[] {
|
||||
return command.map((arg) => compactInlineText(arg, 1200));
|
||||
}
|
||||
|
||||
function compactCommandResult(result: CommandJsonResult): Record<string, unknown> {
|
||||
const data = record(expandedParsedRoot(result).data);
|
||||
const stdout = String(data.stdout ?? result.stdout ?? "");
|
||||
const stderr = String(data.stderr ?? result.stderr ?? "");
|
||||
return {
|
||||
ok: isCommandSuccess(result),
|
||||
exitCode: result.exitCode,
|
||||
command: redactedCommand(result.command),
|
||||
stdout: compactInlineText(stdout.trim(), 4000),
|
||||
stderr: compactInlineText(stderr.trim(), 4000),
|
||||
stdoutBytes: Buffer.byteLength(stdout),
|
||||
stderrBytes: Buffer.byteLength(stderr),
|
||||
};
|
||||
}
|
||||
|
||||
function isCommandSuccess(result: CommandJsonResult): boolean {
|
||||
@@ -410,6 +477,14 @@ function getV02Head(): string | null {
|
||||
return match?.[0] ?? null;
|
||||
}
|
||||
|
||||
function getV02CachedHead(): string | null {
|
||||
const result = v02WorkspaceScript("git rev-parse origin/v0.2", 30_000);
|
||||
if (!isCommandSuccess(result)) return null;
|
||||
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||||
const match = /[0-9a-f]{40}/iu.exec(output);
|
||||
return match?.[0] ?? null;
|
||||
}
|
||||
|
||||
function v02PipelineRunName(sourceCommit: string): string {
|
||||
return `${V02_PIPELINERUN_PREFIX}-${shortSha(sourceCommit)}`;
|
||||
}
|
||||
@@ -425,22 +500,240 @@ function getPipelineRunCompact(name: string): Record<string, unknown> {
|
||||
"-o",
|
||||
"jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}",
|
||||
], 60_000);
|
||||
const text = statusText(result);
|
||||
return pipelineRunCompactFromText(name, statusText(result), isCommandSuccess(result), result.command, result.exitCode, result.stderr);
|
||||
}
|
||||
|
||||
function pipelineRunCompactFromText(
|
||||
name: string,
|
||||
text: string,
|
||||
commandOk: boolean,
|
||||
command: string[] | string,
|
||||
exitCode: number | null,
|
||||
stderr: string,
|
||||
): Record<string, unknown> {
|
||||
const [status = "", reason = "", message = ""] = text.split(/\r?\n/u);
|
||||
const notFound = !isCommandSuccess(result) && /not found/iu.test(`${result.stdout}\n${result.stderr}`);
|
||||
const notFound = !commandOk && /not found/iu.test(`${text}\n${stderr}`);
|
||||
return {
|
||||
ok: isCommandSuccess(result),
|
||||
exists: isCommandSuccess(result) || !notFound,
|
||||
ok: commandOk,
|
||||
exists: commandOk || !notFound,
|
||||
pipelineRun: name,
|
||||
status: status || null,
|
||||
reason: reason || null,
|
||||
message: message || null,
|
||||
command: result.command,
|
||||
exitCode: result.exitCode,
|
||||
stderr: result.stderr.trim().slice(0, 2000),
|
||||
command: Array.isArray(command) ? redactedCommand(command) : command,
|
||||
exitCode,
|
||||
stderr: stderr.trim().slice(0, 2000),
|
||||
};
|
||||
}
|
||||
|
||||
function timestampMs(value: unknown): number | null {
|
||||
if (typeof value !== "string" || value.trim().length === 0) return null;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function secondsBetween(start: unknown, end: unknown): number | null {
|
||||
const startMs = timestampMs(start);
|
||||
const endMs = timestampMs(end);
|
||||
if (startMs === null || endMs === null || endMs < startMs) return null;
|
||||
return Math.round((endMs - startMs) / 1000);
|
||||
}
|
||||
|
||||
function pipelineRunStatusRow(item: unknown, nowMs: number): Record<string, unknown> {
|
||||
const root = record(item);
|
||||
const metadata = record(root.metadata);
|
||||
const status = record(root.status);
|
||||
const labels = record(metadata.labels);
|
||||
const conditions = Array.isArray(status.conditions) ? status.conditions : [];
|
||||
const condition = record(conditions[0]);
|
||||
const startTime = status.startTime ?? null;
|
||||
const completionTime = status.completionTime ?? null;
|
||||
const startMs = timestampMs(startTime);
|
||||
const conditionStatus = typeof condition.status === "string" ? condition.status : null;
|
||||
return {
|
||||
name: metadata.name ?? null,
|
||||
sourceCommit: labels["hwlab.pikastech.local/source-commit"] ?? null,
|
||||
status: conditionStatus,
|
||||
reason: condition.reason ?? null,
|
||||
createdAt: metadata.creationTimestamp ?? null,
|
||||
startTime,
|
||||
completionTime,
|
||||
durationSeconds: secondsBetween(startTime, completionTime),
|
||||
elapsedSeconds: startMs === null || completionTime !== null ? null : Math.max(0, Math.round((nowMs - startMs) / 1000)),
|
||||
active: completionTime === null && conditionStatus !== "True" && conditionStatus !== "False",
|
||||
};
|
||||
}
|
||||
|
||||
function pipelineRunStatusRowFromLine(line: string, nowMs: number): Record<string, unknown> | null {
|
||||
const [
|
||||
name = "",
|
||||
sourceCommit = "",
|
||||
createdAt = "",
|
||||
startTime = "",
|
||||
completionTime = "",
|
||||
status = "",
|
||||
reason = "",
|
||||
] = line.split("\t");
|
||||
if (!name.startsWith(V02_PIPELINERUN_PREFIX)) return null;
|
||||
const startMs = timestampMs(startTime);
|
||||
const completion = completionTime.trim().length > 0 ? completionTime : null;
|
||||
const conditionStatus = status.trim().length > 0 ? status : null;
|
||||
return {
|
||||
name,
|
||||
sourceCommit: sourceCommit || null,
|
||||
status: conditionStatus,
|
||||
reason: reason || null,
|
||||
createdAt: createdAt || null,
|
||||
startTime: startTime || null,
|
||||
completionTime: completion,
|
||||
durationSeconds: secondsBetween(startTime, completion),
|
||||
elapsedSeconds: startMs === null || completion !== null ? null : Math.max(0, Math.round((nowMs - startMs) / 1000)),
|
||||
active: completion === null && conditionStatus !== "True" && conditionStatus !== "False",
|
||||
};
|
||||
}
|
||||
|
||||
function parsePipelineRunRows(text: string, limit: number): Record<string, unknown> {
|
||||
const nowMs = Date.now();
|
||||
const rows = text
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => pipelineRunStatusRowFromLine(line.trim(), nowMs))
|
||||
.filter((item): item is Record<string, unknown> => item !== null)
|
||||
.sort((left, right) => {
|
||||
return (timestampMs(right.createdAt) ?? 0) - (timestampMs(left.createdAt) ?? 0);
|
||||
});
|
||||
const activeItems = rows.filter((item) => item.active === true);
|
||||
return {
|
||||
ok: true,
|
||||
count: rows.length,
|
||||
activeCount: activeItems.length,
|
||||
items: rows.slice(0, limit),
|
||||
activeItems: activeItems.slice(0, limit),
|
||||
disclosure: rows.length > limit ? `showing newest ${limit} of ${rows.length}` : "complete",
|
||||
};
|
||||
}
|
||||
|
||||
function pipelineRunRowsJsonPath(): string {
|
||||
return [
|
||||
"jsonpath={range .items[*]}",
|
||||
"{.metadata.name}",
|
||||
"{\"\\t\"}",
|
||||
"{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}",
|
||||
"{\"\\t\"}",
|
||||
"{.metadata.creationTimestamp}",
|
||||
"{\"\\t\"}",
|
||||
"{.status.startTime}",
|
||||
"{\"\\t\"}",
|
||||
"{.status.completionTime}",
|
||||
"{\"\\t\"}",
|
||||
"{.status.conditions[0].status}",
|
||||
"{\"\\t\"}",
|
||||
"{.status.conditions[0].reason}",
|
||||
"{\"\\n\"}",
|
||||
"{end}",
|
||||
].join("");
|
||||
}
|
||||
|
||||
function listV02PipelineRunsCompact(limit = 8): Record<string, unknown> {
|
||||
const result = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pipelinerun",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
"-l",
|
||||
"hwlab.pikastech.local/gitops-target=v02",
|
||||
"-o",
|
||||
pipelineRunRowsJsonPath(),
|
||||
], 60_000);
|
||||
if (!isCommandSuccess(result)) {
|
||||
return {
|
||||
ok: false,
|
||||
command: redactedCommand(result.command),
|
||||
exitCode: result.exitCode,
|
||||
stderr: commandErrorSummary(result),
|
||||
items: [],
|
||||
activeItems: [],
|
||||
};
|
||||
}
|
||||
return parsePipelineRunRows(statusText(result), limit);
|
||||
}
|
||||
|
||||
function parseShellSections(output: string): Record<string, ShellSection> {
|
||||
const sections: Record<string, ShellSection> = {};
|
||||
let currentName: string | null = null;
|
||||
let currentLines: string[] = [];
|
||||
let currentExitCode: number | null = null;
|
||||
const flush = (): void => {
|
||||
if (currentName === null) return;
|
||||
sections[currentName] = {
|
||||
stdout: currentLines.join("\n").trim(),
|
||||
exitCode: currentExitCode,
|
||||
};
|
||||
};
|
||||
for (const line of output.split(/\r?\n/u)) {
|
||||
const begin = /^__UNIDESK_SECTION_BEGIN__ ([A-Za-z0-9_-]+)$/u.exec(line);
|
||||
if (begin !== null) {
|
||||
flush();
|
||||
currentName = begin[1] ?? null;
|
||||
currentLines = [];
|
||||
currentExitCode = null;
|
||||
continue;
|
||||
}
|
||||
const end = /^__UNIDESK_SECTION_END__ ([A-Za-z0-9_-]+) exit=([0-9]+)$/u.exec(line);
|
||||
if (end !== null && currentName === end[1]) {
|
||||
currentExitCode = Number(end[2]);
|
||||
flush();
|
||||
currentName = null;
|
||||
currentLines = [];
|
||||
currentExitCode = null;
|
||||
continue;
|
||||
}
|
||||
if (currentName !== null) currentLines.push(line);
|
||||
}
|
||||
flush();
|
||||
return sections;
|
||||
}
|
||||
|
||||
function shellSectionOk(section: ShellSection | undefined): boolean {
|
||||
return section?.exitCode === 0;
|
||||
}
|
||||
|
||||
function v02ControlPlaneStatusBundle(pipelineRun: string | null): CommandJsonResult {
|
||||
const script = [
|
||||
"set +e",
|
||||
"section() {",
|
||||
" name=\"$1\"",
|
||||
" shift",
|
||||
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
|
||||
" \"$@\"",
|
||||
" code=$?",
|
||||
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
|
||||
"}",
|
||||
`section controlPlane kubectl get pipeline,role,rolebinding,serviceaccount -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o name`,
|
||||
`section obsoleteCronJobs kubectl get cronjob -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(V02_POLLER)} ${shellQuote(V02_RECONCILER)} --ignore-not-found -o name`,
|
||||
`section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(V02_APP)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`,
|
||||
pipelineRun === null
|
||||
? "section pipelineRun sh -c 'true'"
|
||||
: `section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(pipelineRun)} -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'`,
|
||||
`section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(pipelineRunRowsJsonPath())}`,
|
||||
].join("\n");
|
||||
return g14K3s(["script", "--", script], 60_000);
|
||||
}
|
||||
|
||||
function listV02PipelineRunsCompactFromText(text: string, commandOk: boolean, command: string[] | string, exitCode: number | null, stderr: string, limit = 8): Record<string, unknown> {
|
||||
if (!commandOk) {
|
||||
return {
|
||||
ok: false,
|
||||
command: Array.isArray(command) ? redactedCommand(command) : command,
|
||||
exitCode,
|
||||
stderr: stderr.trim().slice(0, 4000),
|
||||
items: [],
|
||||
activeItems: [],
|
||||
};
|
||||
}
|
||||
return parsePipelineRunRows(text, limit);
|
||||
}
|
||||
|
||||
function pipelinePrefixesForLane(lane: "v02" | "g14" | "all"): string[] {
|
||||
if (lane === "v02") return ["hwlab-v02-ci-poll-"];
|
||||
if (lane === "g14") return ["hwlab-g14-ci-poll-"];
|
||||
@@ -774,8 +1067,8 @@ function refreshV02ControlPlaneBeforeTrigger(sourceCommit: string, timeoutSecond
|
||||
sourceCommit,
|
||||
renderDir: v02RenderDir(sourceCommit),
|
||||
render: commandData(render),
|
||||
apply,
|
||||
cleanupObsoleteCronJobs,
|
||||
apply: compactCommandResult(apply),
|
||||
cleanupObsoleteCronJobs: cleanupObsoleteCronJobs === null ? null : compactCommandResult(cleanupObsoleteCronJobs),
|
||||
degradedReason: !isCommandSuccess(apply)
|
||||
? "control-plane-apply-failed"
|
||||
: cleanupObsoleteCronJobs !== null && !isCommandSuccess(cleanupObsoleteCronJobs)
|
||||
@@ -884,36 +1177,29 @@ function deleteV02PipelineRun(pipelineRun: string): CommandJsonResult {
|
||||
return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, pipelineRun, "--ignore-not-found=true"], 60_000);
|
||||
}
|
||||
|
||||
function v02ControlPlaneStatus(sourceCommit: string | null = getV02Head()): Record<string, unknown> {
|
||||
function v02ControlPlaneStatus(sourceCommit: string | null = getV02CachedHead()): Record<string, unknown> {
|
||||
const pipelineRun = sourceCommit === null ? null : v02PipelineRunName(sourceCommit);
|
||||
const controlPlane = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pipeline,role,rolebinding,serviceaccount",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
"-l",
|
||||
"hwlab.pikastech.local/gitops-target=v02",
|
||||
"-o",
|
||||
"name",
|
||||
], 60_000);
|
||||
const obsoleteCronJobs = getV02ObsoleteCronJobs();
|
||||
const argo = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"application",
|
||||
"-n",
|
||||
ARGO_NAMESPACE,
|
||||
V02_APP,
|
||||
"-o",
|
||||
"jsonpath={.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}",
|
||||
], 60_000);
|
||||
const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = statusText(argo).split(/\r?\n/u);
|
||||
const bundle = v02ControlPlaneStatusBundle(pipelineRun);
|
||||
const sections = parseShellSections(statusText(bundle));
|
||||
const controlPlane = sections.controlPlane;
|
||||
const obsoleteCronJobs = sections.obsoleteCronJobs;
|
||||
const argo = sections.argo;
|
||||
const pipelineRunSection = sections.pipelineRun;
|
||||
const recentPipelineRuns = listV02PipelineRunsCompactFromText(
|
||||
sections.recentPipelineRuns?.stdout ?? "",
|
||||
shellSectionOk(sections.recentPipelineRuns),
|
||||
"kubectl get pipelinerun -n hwlab-ci -l hwlab.pikastech.local/gitops-target=v02 -o jsonpath=<summary-fields>",
|
||||
sections.recentPipelineRuns?.exitCode ?? null,
|
||||
bundle.stderr,
|
||||
);
|
||||
const activePipelineRuns = Array.isArray(recentPipelineRuns.activeItems) ? recentPipelineRuns.activeItems : [];
|
||||
const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = String(argo?.stdout ?? "").split(/\r?\n/u);
|
||||
return {
|
||||
ok: sourceCommit !== null && isCommandSuccess(controlPlane) && isCommandSuccess(argo),
|
||||
ok: sourceCommit !== null && isCommandSuccess(bundle) && shellSectionOk(controlPlane) && shellSectionOk(argo),
|
||||
command: "hwlab g14 control-plane status --lane v02",
|
||||
lane: "v02",
|
||||
sourceCommit,
|
||||
sourceCommitSource: "cached origin/v0.2; write operations fetch before acting",
|
||||
expected: {
|
||||
workspace: V02_WORKSPACE,
|
||||
branch: V02_SOURCE_BRANCH,
|
||||
@@ -924,29 +1210,49 @@ function v02ControlPlaneStatus(sourceCommit: string | null = getV02Head()): Reco
|
||||
argoApplication: V02_APP,
|
||||
},
|
||||
controlPlane: {
|
||||
ok: isCommandSuccess(controlPlane),
|
||||
names: statusText(controlPlane).split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
||||
stderr: controlPlane.stderr.trim().slice(0, 2000),
|
||||
ok: shellSectionOk(controlPlane),
|
||||
names: String(controlPlane?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
||||
exitCode: controlPlane?.exitCode ?? null,
|
||||
},
|
||||
obsoleteCronJobs: {
|
||||
ok: isCommandSuccess(obsoleteCronJobs),
|
||||
names: statusText(obsoleteCronJobs).split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
||||
ok: shellSectionOk(obsoleteCronJobs),
|
||||
names: String(obsoleteCronJobs?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
||||
exitCode: obsoleteCronJobs?.exitCode ?? null,
|
||||
cleanupCommand: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm",
|
||||
},
|
||||
argo: {
|
||||
ok: isCommandSuccess(argo),
|
||||
raw: statusText(argo),
|
||||
ok: shellSectionOk(argo),
|
||||
raw: argo?.stdout ?? "",
|
||||
fields: { targetRevision, path, syncRevision, syncStatus, health },
|
||||
stderr: argo.stderr.trim().slice(0, 2000),
|
||||
exitCode: argo?.exitCode ?? null,
|
||||
},
|
||||
pipelineRun: pipelineRun === null ? null : getPipelineRunCompact(pipelineRun),
|
||||
pipelineRun: pipelineRun === null
|
||||
? null
|
||||
: pipelineRunCompactFromText(
|
||||
pipelineRun,
|
||||
pipelineRunSection?.stdout ?? "",
|
||||
shellSectionOk(pipelineRunSection),
|
||||
`kubectl get pipelinerun -n hwlab-ci ${pipelineRun}`,
|
||||
pipelineRunSection?.exitCode ?? null,
|
||||
bundle.stderr,
|
||||
),
|
||||
activePipelineRuns,
|
||||
recentPipelineRuns,
|
||||
query: {
|
||||
ok: isCommandSuccess(bundle),
|
||||
exitCode: bundle.exitCode,
|
||||
stderr: bundle.stderr.trim().slice(0, 2000),
|
||||
},
|
||||
visibilityHint: activePipelineRuns.length > 0
|
||||
? "activePipelineRuns shows running v02 CI even when origin/v0.2 advanced after a previous trigger"
|
||||
: "no active v02 PipelineRun observed",
|
||||
};
|
||||
}
|
||||
|
||||
function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||||
if (options.action === "cleanup-runs") return runControlPlaneCleanup(options);
|
||||
if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options);
|
||||
const sourceCommit = getV02Head();
|
||||
const sourceCommit = options.action === "status" ? getV02CachedHead() : getV02Head();
|
||||
if (sourceCommit === null) {
|
||||
return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", workspace: V02_WORKSPACE };
|
||||
}
|
||||
@@ -960,7 +1266,7 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
|
||||
command: `hwlab g14 control-plane ${options.action} --lane v02`,
|
||||
phase: "source-render",
|
||||
sourceCommit,
|
||||
render,
|
||||
render: compactCommandResult(render),
|
||||
};
|
||||
}
|
||||
const apply = applyV02ControlPlaneFiles(sourceCommit, options.dryRun, options.timeoutSeconds);
|
||||
@@ -972,8 +1278,8 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
|
||||
sourceCommit,
|
||||
renderDir: v02RenderDir(sourceCommit),
|
||||
render: commandData(render),
|
||||
apply,
|
||||
cleanupObsoleteCronJobs: options.dryRun ? deleteV02ObsoleteCronJobs(true) : deleteV02ObsoleteCronJobs(false),
|
||||
apply: compactCommandResult(apply),
|
||||
cleanupObsoleteCronJobs: compactCommandResult(options.dryRun ? deleteV02ObsoleteCronJobs(true) : deleteV02ObsoleteCronJobs(false)),
|
||||
status: v02ControlPlaneStatus(sourceCommit),
|
||||
next: options.dryRun
|
||||
? { apply: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm" }
|
||||
@@ -1066,8 +1372,8 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
|
||||
before,
|
||||
controlPlaneRefresh,
|
||||
gitMirrorPreSync,
|
||||
deletePipelineRun,
|
||||
createPipelineRun,
|
||||
deletePipelineRun: compactCommandResult(deletePipelineRun),
|
||||
createPipelineRun: createPipelineRun === null ? null : compactCommandResult(createPipelineRun),
|
||||
after: getPipelineRunCompact(v02PipelineRunName(sourceCommit)),
|
||||
};
|
||||
}
|
||||
@@ -1239,6 +1545,8 @@ function compactGitMirrorSync(sync: Record<string, unknown>): Record<string, unk
|
||||
exitCode: nested(sync, ["result", "exitCode"]) ?? null,
|
||||
stdoutTail: tailText(nested(sync, ["result", "stdout"]), 3000),
|
||||
stderrTail: tailText(nested(sync, ["result", "stderr"]), 3000),
|
||||
stdoutBytes: nested(sync, ["result", "stdoutBytes"]) ?? null,
|
||||
stderrBytes: nested(sync, ["result", "stderrBytes"]) ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1390,7 +1698,7 @@ function runGitMirrorApply(options: G14GitMirrorOptions): Record<string, unknown
|
||||
command: "hwlab g14 git-mirror apply",
|
||||
phase: "source-render",
|
||||
sourceCommit,
|
||||
render,
|
||||
render: compactCommandResult(render),
|
||||
};
|
||||
}
|
||||
const apply = applyGitMirrorManifestFile(sourceCommit, options.dryRun, options.timeoutSeconds);
|
||||
@@ -1411,8 +1719,8 @@ function runGitMirrorApply(options: G14GitMirrorOptions): Record<string, unknown
|
||||
sourceCommit,
|
||||
manifest: `${v02RenderDir(sourceCommit)}/devops-infra/git-mirror.yaml`,
|
||||
render: commandData(render),
|
||||
apply,
|
||||
cleanupLegacyCronJob,
|
||||
apply: compactCommandResult(apply),
|
||||
cleanupLegacyCronJob: compactCommandResult(cleanupLegacyCronJob),
|
||||
status: runGitMirrorStatus(),
|
||||
next: options.dryRun
|
||||
? { apply: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm" }
|
||||
@@ -1463,7 +1771,7 @@ function runGitMirrorSync(options: G14GitMirrorOptions): Record<string, unknown>
|
||||
namespace: GIT_MIRROR_NAMESPACE,
|
||||
jobName,
|
||||
manifest: options.dryRun ? manifest : undefined,
|
||||
result,
|
||||
result: compactCommandResult(result),
|
||||
status: options.dryRun ? undefined : runGitMirrorStatus(),
|
||||
next: options.dryRun ? { sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm" } : { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
|
||||
};
|
||||
@@ -1512,7 +1820,7 @@ function runGitMirrorFlush(options: G14GitMirrorOptions): Record<string, unknown
|
||||
namespace: GIT_MIRROR_NAMESPACE,
|
||||
jobName,
|
||||
manifest: options.dryRun ? manifest : undefined,
|
||||
result,
|
||||
result: compactCommandResult(result),
|
||||
status: options.dryRun ? undefined : runGitMirrorStatus(),
|
||||
next: options.dryRun ? { flush: "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm" } : { status: "bun scripts/cli.ts hwlab g14 git-mirror status" },
|
||||
};
|
||||
@@ -1527,7 +1835,7 @@ function runG14GitMirror(options: G14GitMirrorOptions): Record<string, unknown>
|
||||
|
||||
function startAsyncHwlabG14Job(name: string, command: string[], note: string): Record<string, unknown> {
|
||||
const job = startJob(name, command, note);
|
||||
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 30000`;
|
||||
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`;
|
||||
return {
|
||||
ok: true,
|
||||
mode: "async-job",
|
||||
|
||||
Reference in New Issue
Block a user