feat: add hwlab v02 control-plane cli
This commit is contained in:
@@ -43,6 +43,7 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 DEV/PROD 滚动、P
|
||||
- `artifact-registry plan|render|status|health|install|deploy-backend-core|deploy-service` 管理 D601 host-managed CNCF Distribution registry 的声明、安装、只读检查和 pull-only artifact CD。该 registry 固定为 D601 loopback `127.0.0.1:5000`,由 systemd + Docker Compose 管理,位于 native k3s 故障域外;`deploy-service` 只拉取 CI 已发布的 commit-pinned 镜像、retag/recreate 或导入 native k3s,并做 live commit 验证,不构建 runtime source。`deploy-backend-core` 是 deprecated 兼容名,标准 backend-core prod CD 入口是 `deploy apply --env prod --service backend-core`。长期规则见 `docs/reference/artifact-registry.md`。
|
||||
- `commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr` 是 host Codex 指挥官直管微服务 skeleton 入口。当前命令返回 `phase=source-contract`、service/API/state/bridge/prompt/trace/#20/#46/ClaudeQQ 审批边界、.state/commander/ 状态模型、dev 无 daemon smoke contract、dry-run 计划和 GPT-5.5 PR prompt 边界辅助 lint,不接 live bridge、不注入 prompt、不发送 ClaudeQQ。`approval request --dry-run` 会生成 200 字以内中文纯文本 ClaudeQQ 审批草案、`notification-path-unavailable` blocker 和授权后唯一可用的 `bun scripts/cli.ts microservice proxy claudeqq /api/push/text --method POST --body-json '<payload>' --raw` 命令;不得提示使用本机 ClaudeQQ skill、powershell 或本地 server。`prompt-lint` 支持 `--prompt-file` 与 `--stdin`,输出 `ok`、`missingClauses`、`riskLevel`、`suggestedPatchSnippet` 且不回显完整 prompt;它是 commander 辅助检查,不是业务 PR 门禁,也不改变 `codex submit` 默认行为。`plan`、`smoke` 与 `approval request` 必须带 `--dry-run`;缺少时返回 `error=dry-run-required`。长期规则见 `docs/reference/host-codex-commander.md`。
|
||||
- `hwlab g14 monitor-prs [--once] [--dry-run] [--interval-seconds N] [--max-cycles N] [--timeout-seconds N]` 是当前 HWLAB G14 PR -> CI/CD -> DEV rollout 的一行式入口。普通调用创建 `.state/jobs/` 异步 job 并立刻返回 `job.id`、`statusCommand` 和 stdout/stderr 路径;后台 worker 每轮通过 UniDesk `gh pr list/preflight/merge` 监控 `pikasTech/HWLAB` base=`G14` 的 open PR,ready 时合并,然后通过 UniDesk `ssh G14:k3s` 观察 `hwlab-g14-ci-poll-<short>`、Argo `hwlab-g14-dev` 和 DEV `/health/live`,直到 DEV `Synced/Healthy` 且 Deployment/StatefulSet ready;历史 `Completed` smoke/debug pod 不作为 rollout blocker。每次成功 DEV rollout 后,worker 会定位或创建 #7“指挥简报索引”中的北京日期每日简报 issue,并追加 CI/CD 耗时、CI/CD 关键指标、语义化上线 changelog、自动 diff 摘要、PipelineRun、GitOps revision 和 DEV 验证摘要;关键指标来自 G14 Tekton TaskRun results,固定包含 `lazy build reused: x/y`、reused services、rebuild services 和每个 service 的独立耗时/状态/backend,用于观察 lazy build 机制效果。语义化 changelog 优先从 PR body 的 `## 修改`/`## 变更`/`## Changelog` 等段落提取,diff 摘要只作为文件和统计证据保留,不替代 changelog。也可用 `hwlab g14 record-rollout --pr <number> --source-commit <sha>` 手动补记,手动补记同样会按 PipelineRun 采集 TaskRun 指标。状态指针按用途分离:长期监控只写 `.state/hwlab-g14/latest-monitor-job.json`,`--once` 写 `latest-once-job.json`,`--dry-run` 写 `latest-dry-run-job.json`,`--once --dry-run` 写 `latest-once-dry-run-job.json`,避免一次性收口覆盖持续监控入口。`--once --dry-run` 只做单轮监控和 merge plan,不写 GitHub、不等待 rollout。该命令禁止使用原生 `gh` 或手拼 GitHub 请求;如果 UniDesk `gh` 子命令字段或行为不够,必须先改进 `scripts/src/gh.ts` 后再使用。
|
||||
- `hwlab g14 control-plane status|apply|rerun-current --lane v02 [--dry-run|--confirm]` 是 HWLAB `v0.2` 加法 lane 的受控 Tekton/Argo 控制面维护入口,只面向 G14 `/root/hwlab-v02`、branch `v0.2`、namespace `hwlab-ci` 和 Argo application `hwlab-g14-v02`;`status` 只读汇总 `hwlab-v02-ci-image-publish`、`hwlab-v02-branch-poller`、`hwlab-v02-control-plane-reconciler`、`hwlab-g14-v02` 和当前 commit PipelineRun。`apply` 先在 G14 `/root/hwlab-v02` 快进并执行 `scripts/g14-gitops-render.mjs --lane v02 --check`,再只通过 UniDesk `G14:k3s` route server-side apply `tekton-v02/rbac.yaml`、`pipeline.yaml`、`poller.yaml`、`control-plane-reconciler.yaml`、`argocd/project.yaml` 和 `argocd/application-v02.yaml`;默认 dry-run,真实写入必须加 `--confirm`,不会应用 runtime-v02 workload、Secret 或数据迁移。`rerun-current` 用于控制面更新后重跑当前 `origin/v0.2` commit:它只在同名 PipelineRun 非成功且非运行时删除旧失败 run 并从 `hwlab-v02-branch-poller` 创建一次性 Job;默认 dry-run,真实动作必须加 `--confirm`。`hwlab g14 tools-image status|build --name ci-node-tools --tag <tag> [--dockerfile deploy/ci/hwlab-ci-node-tools.Dockerfile] [--dry-run|--confirm]` 是 G14 固定 HWLAB CI tools image 的受控 host build/push 入口:镜像内容必须来自 HWLAB repo 内 Dockerfile,`build` 默认 dry-run,真实写入必须加 `--confirm`,构建和 push 只发生在 G14 host 与本地 registry,不在 master server 构建,也不把 `apk add`/runtime install 塞进 Tekton PipelineRun。
|
||||
- `ssh gh:/owner/repo ...` 把 GitHub issue/PR 映射成只读/受控写入的虚拟文本目录,适合日报、PR 正文和 issue 正文的小补丁维护:`ssh gh:/pikasTech/HWLAB ls` 展示 `pr/` 与 `issue/`,`ssh gh:/pikasTech/HWLAB/pr ls [--limit N] [--full]` 和 `ssh gh:/pikasTech/HWLAB/issue ls [--limit N] [--full]` 展示条目状态、楼层数、正文长度和标题,`ssh gh:/pikasTech/HWLAB/pr/507 ls` 展示单个 PR 的一楼正文文件,`ssh gh:/pikasTech/HWLAB/505/1 cat|rg|patch-apply` 兼容旧式 issue/PR number route。`patch-apply` 使用 UniDesk 默认 apply-patch v2 的虚拟文件 executor,把正文一楼映射为 `body.md`,写回仍走 `bun scripts/cli.ts gh issue/pr update` 的 guard/concurrency 规则;`rm` 对正文一楼结构化拒绝,避免误删 issue/PR 正文。大正文读取必须展开 UniDesk gh dump 文件,否则 `cat/rg/patch-apply` 会误读为空,这是 `gh:` 虚拟文件接口的 P0 可见性契约。
|
||||
- `hwlab cd status|audit|preflight|apply --env dev [--dry-run]` 是旧 D601 HWLAB DEV CD 指挥侧 wrapper,仅用于显式 legacy 诊断和迁移对照。默认通过 UniDesk provider `host.ssh` 进入 D601,再调用 HWLAB repo-owned `scripts/dev-cd-apply.mjs`,不内嵌发布 kubectl 逻辑:`status` 汇总固定 CD mirror、Git clean/main/origin-main、`deploy/deploy.json`/artifact catalog/report、D601 native k3s guard 和 CD Lease lock,并用 `scripts/dev-cd-apply.mjs --status --skip-live-verify` 取得 target/promotion 摘要;`audit` 在 k3s/CD 恢复后做只读健康审计,返回有界 JSON 的 blocker 分类、D601 guard/node、SecretRef 存在性、registry 可达性、Lease phase/holder/staleness、deploy.json 与 artifact/workload image 收敛、current Deployment image/revision/rollout、16666/16667 public health commit/readiness 和 DB/runtime durability 摘要;`preflight` 进一步检查必需 SecretRef 对象/键存在性并运行 HWLAB `scripts/dev-cd-apply.mjs --dry-run --skip-live-verify` 受控事务摘要。完整远端 stdout/stderr 写入 D601 `~/.state/unidesk-hwlab-cd/<run-id>/` 和本地 `.state/hwlab-cd/<run-id>/` task dump,stdout 只返回有界摘要。默认 HWLAB CD repo 是 `/home/ubuntu/hwlab_cd`,`/home/ubuntu/hwlab` runner 历史目录不得作为发布真相。wrapper 强制 `KUBECONFIG=/etc/rancher/k3s/k3s.yaml` 并只以这个显式目标作为 gate;显式目标出现 `docker-desktop`、`desktop-control-plane` 或 `127.0.0.1:11700` 信号会结构化拒绝,audit/preflight/apply --dry-run 都必须观察到 node `d601`。真实 apply 只暴露 `scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report` 命令形状并标注 host-commander-only,本 runner 不执行 live apply、rollout、Lease mutation 或 DEV deploy apply。长期规则见 `docs/reference/hwlab.md`。
|
||||
- `gh auth status [--repo owner/name]` 探测 GitHub 操作前置条件并输出脱敏 JSON:是否存在 `gh` binary、是否存在 `GH_TOKEN`/`GITHUB_TOKEN` 或可用 `gh auth token` fallback、REST API 是否可达、目标 repo 是否可见、issue 是否可读。degraded reason 必须归类为 `missing-binary`、`missing-token`、`auth-failed`、`github-transient`、`network-proxy-failed`、`permission-denied`、`repo-not-found`、`repo-forbidden`、`issue-not-found`、`pr-not-found`、`scope-insufficient`、`validation-failed`、`invalid-response` 或 `unsupported-command`,不得打印 token;失败对象必须包含 `runnerDisposition=infra-blocked|business-failed`,runner 应优先用该字段分流。`github-transient` 表示 GitHub DNS/API 连接在收到 HTTP 状态前失败,输出应带 `retryable=true` 或等价 commander action;这不是缺 token、认证失败、权限不足或 PR 语义失败。
|
||||
|
||||
+1
-1
@@ -57,7 +57,7 @@ export function rootHelp(): unknown {
|
||||
{ command: "auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run", description: "Inspect the P0 Rust auth broker and CLI adapter contract without reading token values, writing GitHub, or starting services." },
|
||||
{ command: "gh preflight|auth|issue|pr", description: "Run safe GitHub issue and PR CRUD/lifecycle operations through REST with body-file update replace/append, comment delete, token diagnostics, PR closeout preflight, hard delete unsupported, and guarded PR merge." },
|
||||
{ command: "commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr", description: "Host Codex commander skeleton contract, no-daemon smoke plan, dry-run approval preview, and advisory GPT-5.5 PR prompt boundary lint without live bridges, message sends, or submit gating." },
|
||||
{ command: "hwlab g14 monitor-prs", description: "Start a fire-and-forget monitor that watches HWLAB PRs targeting G14, merges ready PRs through UniDesk gh, waits for G14 Tekton/GitOps/Argo DEV rollout, and appends the #7-indexed daily brief." },
|
||||
{ command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|rerun-current --lane v02 | hwlab g14 tools-image status|build", description: "Start the G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane actions, or build/status fixed HWLAB CI tools images through UniDesk G14 routes." },
|
||||
{ command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Legacy D601 HWLAB DEV CD wrapper kept for explicit old-path diagnostics; current HWLAB rollout uses G14 GitOps." },
|
||||
{ command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." },
|
||||
{ command: "schedule list|get|runs|run|retry-run|delete", description: "Manage backend-core scheduled tasks and run history; schedule run <id> supports --wait-ms N and retry-run reuses the failed run's schedule." },
|
||||
|
||||
+496
-2
@@ -8,10 +8,20 @@ const HWLAB_REPO = "pikasTech/HWLAB";
|
||||
const G14_SOURCE_BRANCH = "G14";
|
||||
const G14_PROVIDER = "G14";
|
||||
const G14_WORKSPACE = "/root/hwlab";
|
||||
const V02_SOURCE_BRANCH = "v0.2";
|
||||
const V02_WORKSPACE = "/root/hwlab-v02";
|
||||
const DEV_NAMESPACE = "hwlab-dev";
|
||||
const CI_NAMESPACE = "hwlab-ci";
|
||||
const ARGO_NAMESPACE = "argocd";
|
||||
const DEV_APP = "hwlab-g14-dev";
|
||||
const V02_APP = "hwlab-g14-v02";
|
||||
const V02_PIPELINE = "hwlab-v02-ci-image-publish";
|
||||
const V02_POLLER = "hwlab-v02-branch-poller";
|
||||
const V02_RECONCILER = "hwlab-v02-control-plane-reconciler";
|
||||
const V02_PIPELINERUN_PREFIX = "hwlab-v02-ci-poll";
|
||||
const V02_CONTROL_PLANE_FIELD_MANAGER = "unidesk-hwlab-v02-control-plane";
|
||||
const G14_CI_TOOLS_IMAGE_REPO = "127.0.0.1:5000/hwlab/hwlab-ci-node-tools";
|
||||
const G14_CI_TOOLS_BASE_TAG = "node22-alpine-v1";
|
||||
const DEFAULT_INTERVAL_SECONDS = 600;
|
||||
const DEFAULT_MAX_CYCLES = 0;
|
||||
const DEFAULT_TIMEOUT_SECONDS = 1800;
|
||||
@@ -38,6 +48,24 @@ interface G14RecordRolloutOptions {
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
interface G14ControlPlaneOptions {
|
||||
action: "status" | "apply" | "rerun-current";
|
||||
lane: "v02";
|
||||
dryRun: boolean;
|
||||
confirm: boolean;
|
||||
timeoutSeconds: number;
|
||||
}
|
||||
|
||||
interface G14ToolsImageOptions {
|
||||
action: "status" | "build";
|
||||
name: "ci-node-tools";
|
||||
tag: string;
|
||||
dockerfile: string;
|
||||
dryRun: boolean;
|
||||
confirm: boolean;
|
||||
timeoutSeconds: number;
|
||||
}
|
||||
|
||||
interface CommandJsonResult {
|
||||
ok: boolean;
|
||||
command: string[];
|
||||
@@ -131,6 +159,50 @@ function parseRecordRolloutOptions(args: string[]): G14RecordRolloutOptions {
|
||||
};
|
||||
}
|
||||
|
||||
function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions {
|
||||
const [actionRaw] = args;
|
||||
if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "rerun-current") {
|
||||
throw new Error("control-plane usage: status|apply|rerun-current --lane v02 [--dry-run|--confirm]");
|
||||
}
|
||||
const lane = optionValue(args, "--lane");
|
||||
if (lane !== "v02") throw new Error("control-plane currently requires --lane v02");
|
||||
const confirm = args.includes("--confirm");
|
||||
const explicitDryRun = args.includes("--dry-run");
|
||||
if (confirm && explicitDryRun) throw new Error("control-plane accepts only one of --confirm or --dry-run");
|
||||
return {
|
||||
action: actionRaw,
|
||||
lane: "v02",
|
||||
confirm,
|
||||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
||||
};
|
||||
}
|
||||
|
||||
function parseToolsImageOptions(args: string[]): G14ToolsImageOptions {
|
||||
const [actionRaw] = args;
|
||||
if (actionRaw !== "status" && actionRaw !== "build") {
|
||||
throw new Error("tools-image usage: status|build --name ci-node-tools --tag <tag> [--dockerfile path] [--dry-run|--confirm]");
|
||||
}
|
||||
const name = optionValue(args, "--name") ?? "ci-node-tools";
|
||||
if (name !== "ci-node-tools") throw new Error("tools-image currently supports --name ci-node-tools");
|
||||
const tag = optionValue(args, "--tag") ?? G14_CI_TOOLS_BASE_TAG;
|
||||
if (!/^[A-Za-z0-9_.-]+$/u.test(tag)) throw new Error("--tag may only contain letters, numbers, dot, underscore, and dash");
|
||||
const dockerfile = optionValue(args, "--dockerfile") ?? "deploy/ci/hwlab-ci-node-tools.Dockerfile";
|
||||
if (dockerfile.startsWith("/") || dockerfile.includes("..")) throw new Error("--dockerfile must be a repo-relative path without '..'");
|
||||
const confirm = args.includes("--confirm");
|
||||
const explicitDryRun = args.includes("--dry-run");
|
||||
if (confirm && explicitDryRun) throw new Error("tools-image accepts only one of --confirm or --dry-run");
|
||||
return {
|
||||
action: actionRaw,
|
||||
name,
|
||||
tag,
|
||||
dockerfile,
|
||||
confirm,
|
||||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 600, 1800),
|
||||
};
|
||||
}
|
||||
|
||||
function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
|
||||
const index = args.indexOf(name);
|
||||
if (index === -1) return defaultValue;
|
||||
@@ -140,6 +212,10 @@ function positiveIntegerOption(args: string[], name: string, defaultValue: numbe
|
||||
return Math.min(value, maxValue);
|
||||
}
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replace(/'/gu, `'"'"'`)}'`;
|
||||
}
|
||||
|
||||
function commandJson(command: string[], timeoutMs = 60_000): CommandJsonResult {
|
||||
const result = runCommand(command, repoRoot, { timeoutMs });
|
||||
let parsed: unknown | null = null;
|
||||
@@ -233,6 +309,407 @@ function precheckWorkspace(): CommandJsonResult {
|
||||
return cliJson(["ssh", `${G14_PROVIDER}:${G14_WORKSPACE}`, "script", "--", "pwd; git fetch origin G14 --prune; git status --short --branch; git remote -v | sed -n '1,4p'"], 120_000);
|
||||
}
|
||||
|
||||
function v02WorkspaceScript(script: string, timeoutMs = 120_000): CommandJsonResult {
|
||||
return cliJson(["ssh", `${G14_PROVIDER}:${V02_WORKSPACE}`, "script", "--", script], timeoutMs);
|
||||
}
|
||||
|
||||
function g14K3s(args: string[], timeoutMs = 60_000): CommandJsonResult {
|
||||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, ...args], timeoutMs);
|
||||
}
|
||||
|
||||
function getV02Head(): string | null {
|
||||
const result = v02WorkspaceScript("git fetch origin v0.2 --prune >/dev/null 2>&1; git rev-parse origin/v0.2", 120_000);
|
||||
if (!isCommandSuccess(result)) return null;
|
||||
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||||
const match = /[0-9a-f]{40}/iu.exec(output);
|
||||
return match?.[0] ?? null;
|
||||
}
|
||||
|
||||
function v02PipelineRunName(sourceCommit: string): string {
|
||||
return `${V02_PIPELINERUN_PREFIX}-${shortSha(sourceCommit)}`;
|
||||
}
|
||||
|
||||
function v02ManualPollerJobName(sourceCommit: string): string {
|
||||
return `hwlab-v02-branch-poller-manual-${shortSha(sourceCommit)}`;
|
||||
}
|
||||
|
||||
function getPipelineRunCompact(name: string): Record<string, unknown> {
|
||||
const result = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pipelinerun",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
name,
|
||||
"-o",
|
||||
"jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}",
|
||||
], 60_000);
|
||||
const text = statusText(result);
|
||||
const [status = "", reason = "", message = ""] = text.split(/\r?\n/u);
|
||||
const notFound = !isCommandSuccess(result) && /not found/iu.test(`${result.stdout}\n${result.stderr}`);
|
||||
return {
|
||||
ok: isCommandSuccess(result),
|
||||
exists: isCommandSuccess(result) || !notFound,
|
||||
pipelineRun: name,
|
||||
status: status || null,
|
||||
reason: reason || null,
|
||||
message: message || null,
|
||||
command: result.command,
|
||||
exitCode: result.exitCode,
|
||||
stderr: result.stderr.trim().slice(0, 2000),
|
||||
};
|
||||
}
|
||||
|
||||
function runV02RenderCheck(sourceCommit: string): CommandJsonResult {
|
||||
const renderDir = v02RenderDir(sourceCommit);
|
||||
return v02WorkspaceScript([
|
||||
"set -eu",
|
||||
"git fetch origin v0.2 --prune",
|
||||
"git checkout v0.2 >/dev/null 2>&1 || true",
|
||||
"git merge --ff-only origin/v0.2",
|
||||
`test "$(git rev-parse HEAD)" = ${shellQuote(sourceCommit)}`,
|
||||
`rm -rf ${shellQuote(renderDir)}`,
|
||||
`mkdir -p ${shellQuote(renderDir)}`,
|
||||
`node scripts/g14-gitops-render.mjs --lane v02 --source-revision ${shellQuote(sourceCommit)} --out ${shellQuote(renderDir)}`,
|
||||
`node scripts/g14-gitops-render.mjs --lane v02 --source-revision ${shellQuote(sourceCommit)} --out ${shellQuote(renderDir)} --check`,
|
||||
].join("\n"), 180_000);
|
||||
}
|
||||
|
||||
function v02RenderDir(sourceCommit: string): string {
|
||||
return `/tmp/hwlab-v02-control-plane-${shortSha(sourceCommit)}`;
|
||||
}
|
||||
|
||||
function applyV02ControlPlaneFiles(sourceCommit: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
||||
const renderDir = v02RenderDir(sourceCommit);
|
||||
return g14K3s([
|
||||
"kubectl",
|
||||
"apply",
|
||||
"--server-side",
|
||||
"--force-conflicts",
|
||||
`--field-manager=${V02_CONTROL_PLANE_FIELD_MANAGER}`,
|
||||
...(dryRun ? ["--dry-run=server"] : []),
|
||||
"-f",
|
||||
`${renderDir}/tekton-v02/rbac.yaml`,
|
||||
"-f",
|
||||
`${renderDir}/tekton-v02/pipeline.yaml`,
|
||||
"-f",
|
||||
`${renderDir}/tekton-v02/poller.yaml`,
|
||||
"-f",
|
||||
`${renderDir}/tekton-v02/control-plane-reconciler.yaml`,
|
||||
"-f",
|
||||
`${renderDir}/argocd/project.yaml`,
|
||||
"-f",
|
||||
`${renderDir}/argocd/application-v02.yaml`,
|
||||
], timeoutSeconds * 1000);
|
||||
}
|
||||
|
||||
function getV02PollerCronJob(): CommandJsonResult {
|
||||
return g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"cronjob",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
V02_POLLER,
|
||||
"-o",
|
||||
"jsonpath={.metadata.name}{\"\\n\"}{.spec.schedule}{\"\\n\"}",
|
||||
], 60_000);
|
||||
}
|
||||
|
||||
function deleteV02PipelineRun(pipelineRun: string): CommandJsonResult {
|
||||
return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, pipelineRun, "--ignore-not-found=true"], 60_000);
|
||||
}
|
||||
|
||||
function deleteV02ManualPollerJob(jobName: string): CommandJsonResult {
|
||||
return g14K3s(["kubectl", "delete", "job", "-n", CI_NAMESPACE, jobName, "--ignore-not-found=true"], 60_000);
|
||||
}
|
||||
|
||||
function createV02ManualPollerJob(jobName: string): CommandJsonResult {
|
||||
return g14K3s(["kubectl", "create", "job", "-n", CI_NAMESPACE, `--from=cronjob/${V02_POLLER}`, jobName], 60_000);
|
||||
}
|
||||
|
||||
function getV02ManualPollerJob(jobName: string): CommandJsonResult {
|
||||
return g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"job",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
jobName,
|
||||
"-o",
|
||||
"jsonpath={.metadata.name}{\"\\n\"}{.status.conditions[0].type}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}",
|
||||
], 60_000);
|
||||
}
|
||||
|
||||
function v02ControlPlaneStatus(sourceCommit: string | null = getV02Head()): Record<string, unknown> {
|
||||
const pipelineRun = sourceCommit === null ? null : v02PipelineRunName(sourceCommit);
|
||||
const controlPlane = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"cronjob,pipeline,role,rolebinding,serviceaccount",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
"-l",
|
||||
"hwlab.pikastech.local/gitops-target=v02",
|
||||
"-o",
|
||||
"name",
|
||||
], 60_000);
|
||||
const argo = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"application",
|
||||
"-n",
|
||||
ARGO_NAMESPACE,
|
||||
V02_APP,
|
||||
"-o",
|
||||
"jsonpath={.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}",
|
||||
], 60_000);
|
||||
const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = statusText(argo).split(/\r?\n/u);
|
||||
return {
|
||||
ok: sourceCommit !== null && isCommandSuccess(controlPlane) && isCommandSuccess(argo),
|
||||
command: "hwlab g14 control-plane status --lane v02",
|
||||
lane: "v02",
|
||||
sourceCommit,
|
||||
expected: {
|
||||
workspace: V02_WORKSPACE,
|
||||
branch: V02_SOURCE_BRANCH,
|
||||
namespace: CI_NAMESPACE,
|
||||
runtimeNamespace: "hwlab-v02",
|
||||
pipeline: V02_PIPELINE,
|
||||
poller: V02_POLLER,
|
||||
reconciler: V02_RECONCILER,
|
||||
argoApplication: V02_APP,
|
||||
},
|
||||
controlPlane: {
|
||||
ok: isCommandSuccess(controlPlane),
|
||||
names: statusText(controlPlane).split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
||||
stderr: controlPlane.stderr.trim().slice(0, 2000),
|
||||
},
|
||||
argo: {
|
||||
ok: isCommandSuccess(argo),
|
||||
raw: statusText(argo),
|
||||
fields: { targetRevision, path, syncRevision, syncStatus, health },
|
||||
stderr: argo.stderr.trim().slice(0, 2000),
|
||||
},
|
||||
pipelineRun: pipelineRun === null ? null : getPipelineRunCompact(pipelineRun),
|
||||
};
|
||||
}
|
||||
|
||||
function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||||
const sourceCommit = getV02Head();
|
||||
if (sourceCommit === null) {
|
||||
return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", workspace: V02_WORKSPACE };
|
||||
}
|
||||
if (options.action === "status") return v02ControlPlaneStatus(sourceCommit);
|
||||
if (options.action === "apply") {
|
||||
const renderCheck = runV02RenderCheck(sourceCommit);
|
||||
if (!isCommandSuccess(renderCheck)) {
|
||||
return {
|
||||
ok: false,
|
||||
command: `hwlab g14 control-plane ${options.action} --lane v02`,
|
||||
phase: "source-render-check",
|
||||
sourceCommit,
|
||||
renderCheck,
|
||||
};
|
||||
}
|
||||
const apply = applyV02ControlPlaneFiles(sourceCommit, options.dryRun, options.timeoutSeconds);
|
||||
return {
|
||||
ok: isCommandSuccess(apply),
|
||||
command: "hwlab g14 control-plane apply --lane v02",
|
||||
lane: "v02",
|
||||
mode: options.dryRun ? "dry-run" : "confirmed-apply",
|
||||
sourceCommit,
|
||||
renderDir: v02RenderDir(sourceCommit),
|
||||
renderCheck: commandData(renderCheck),
|
||||
apply,
|
||||
status: v02ControlPlaneStatus(sourceCommit),
|
||||
next: options.dryRun
|
||||
? { apply: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm" }
|
||||
: { rerunCurrent: "bun scripts/cli.ts hwlab g14 control-plane rerun-current --lane v02 --confirm" },
|
||||
};
|
||||
}
|
||||
const before = getPipelineRunCompact(v02PipelineRunName(sourceCommit));
|
||||
if (options.dryRun) {
|
||||
const poller = getV02PollerCronJob();
|
||||
return {
|
||||
ok: isCommandSuccess(poller),
|
||||
command: "hwlab g14 control-plane rerun-current --lane v02",
|
||||
lane: "v02",
|
||||
mode: "dry-run",
|
||||
sourceCommit,
|
||||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||||
manualJob: v02ManualPollerJobName(sourceCommit),
|
||||
before,
|
||||
poller,
|
||||
next: { rerunCurrent: "bun scripts/cli.ts hwlab g14 control-plane rerun-current --lane v02 --confirm" },
|
||||
};
|
||||
}
|
||||
if (before.status === "True" || before.status === "Unknown") {
|
||||
return {
|
||||
ok: false,
|
||||
command: "hwlab g14 control-plane rerun-current --lane v02",
|
||||
lane: "v02",
|
||||
mode: "confirmed-rerun",
|
||||
sourceCommit,
|
||||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||||
before,
|
||||
degradedReason: "refuse-active-or-successful-pipelinerun",
|
||||
};
|
||||
}
|
||||
const deletePipelineRun = deleteV02PipelineRun(v02PipelineRunName(sourceCommit));
|
||||
const deleteJob = deleteV02ManualPollerJob(v02ManualPollerJobName(sourceCommit));
|
||||
const createJob = isCommandSuccess(deletePipelineRun) && isCommandSuccess(deleteJob)
|
||||
? createV02ManualPollerJob(v02ManualPollerJobName(sourceCommit))
|
||||
: null;
|
||||
const job = createJob !== null && isCommandSuccess(createJob) ? getV02ManualPollerJob(v02ManualPollerJobName(sourceCommit)) : null;
|
||||
return {
|
||||
ok: isCommandSuccess(deletePipelineRun) && isCommandSuccess(deleteJob) && createJob !== null && isCommandSuccess(createJob),
|
||||
command: "hwlab g14 control-plane rerun-current --lane v02",
|
||||
lane: "v02",
|
||||
mode: "confirmed-rerun",
|
||||
sourceCommit,
|
||||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||||
manualJob: v02ManualPollerJobName(sourceCommit),
|
||||
before,
|
||||
deletePipelineRun,
|
||||
deleteJob,
|
||||
createJob,
|
||||
job,
|
||||
after: getPipelineRunCompact(v02PipelineRunName(sourceCommit)),
|
||||
};
|
||||
}
|
||||
|
||||
function g14HostScript(script: string, timeoutMs = 120_000): CommandJsonResult {
|
||||
return cliJson(["ssh", G14_PROVIDER, "script", "--", script], timeoutMs);
|
||||
}
|
||||
|
||||
function g14CiToolsImage(tag: string): string {
|
||||
return `${G14_CI_TOOLS_IMAGE_REPO}:${tag}`;
|
||||
}
|
||||
|
||||
function runG14ToolsImageStatus(options: G14ToolsImageOptions): Record<string, unknown> {
|
||||
const image = g14CiToolsImage(options.tag);
|
||||
const script = [
|
||||
"set -u",
|
||||
`image=${shellQuote(image)}`,
|
||||
"exists=false",
|
||||
"image_id=",
|
||||
"created=",
|
||||
"size=",
|
||||
"node_version=",
|
||||
"npm_version=",
|
||||
"bun_version=",
|
||||
"git_version=",
|
||||
"docker_version=",
|
||||
"python_version=",
|
||||
"registry_tags=",
|
||||
"if docker image inspect \"$image\" >/tmp/hwlab-tools-image-inspect.json 2>/tmp/hwlab-tools-image-inspect.err; then",
|
||||
" exists=true",
|
||||
" image_id=$(docker image inspect \"$image\" --format '{{.Id}}' 2>/dev/null || true)",
|
||||
" created=$(docker image inspect \"$image\" --format '{{.Created}}' 2>/dev/null || true)",
|
||||
" size=$(docker image inspect \"$image\" --format '{{.Size}}' 2>/dev/null || true)",
|
||||
" node_version=$(docker run --rm \"$image\" node --version 2>/dev/null || true)",
|
||||
" npm_version=$(docker run --rm \"$image\" npm --version 2>/dev/null || true)",
|
||||
" bun_version=$(docker run --rm \"$image\" bun --version 2>/dev/null || true)",
|
||||
" git_version=$(docker run --rm \"$image\" git --version 2>/dev/null || true)",
|
||||
" docker_version=$(docker run --rm \"$image\" docker --version 2>/dev/null || true)",
|
||||
" python_version=$(docker run --rm \"$image\" python3 --version 2>/dev/null || true)",
|
||||
"fi",
|
||||
`registry_tags=$(curl -fsS --max-time 10 http://127.0.0.1:5000/v2/hwlab/hwlab-ci-node-tools/tags/list 2>/dev/null || true)`,
|
||||
"export image exists image_id created size node_version npm_version bun_version git_version docker_version python_version registry_tags",
|
||||
"node - <<'NODE'",
|
||||
"const fs = require('node:fs');",
|
||||
"const env = process.env;",
|
||||
"const payload = {",
|
||||
" image: env.image,",
|
||||
" exists: env.exists === 'true',",
|
||||
" imageId: env.image_id || null,",
|
||||
" created: env.created || null,",
|
||||
" sizeBytes: env.size ? Number(env.size) : null,",
|
||||
" tools: {",
|
||||
" node: env.node_version || null,",
|
||||
" npm: env.npm_version || null,",
|
||||
" bun: env.bun_version || null,",
|
||||
" git: env.git_version || null,",
|
||||
" docker: env.docker_version || null,",
|
||||
" python: env.python_version || null",
|
||||
" },",
|
||||
" registryTagsRaw: env.registry_tags || null",
|
||||
"};",
|
||||
"console.log(JSON.stringify(payload, null, 2));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
const result = g14HostScript(script, 180_000);
|
||||
let parsedStatus: unknown = null;
|
||||
const text = statusText(result);
|
||||
if (text.length > 0) {
|
||||
try {
|
||||
parsedStatus = JSON.parse(text) as unknown;
|
||||
} catch {
|
||||
parsedStatus = null;
|
||||
}
|
||||
}
|
||||
return {
|
||||
ok: isCommandSuccess(result) && record(parsedStatus).exists === true,
|
||||
command: "hwlab g14 tools-image status --name ci-node-tools",
|
||||
name: options.name,
|
||||
image,
|
||||
status: parsedStatus ?? text,
|
||||
result,
|
||||
};
|
||||
}
|
||||
|
||||
function runG14ToolsImageBuild(options: G14ToolsImageOptions): Record<string, unknown> {
|
||||
const image = g14CiToolsImage(options.tag);
|
||||
const script = [
|
||||
"set -eu",
|
||||
`cd ${shellQuote(V02_WORKSPACE)}`,
|
||||
"git fetch origin v0.2 --prune",
|
||||
"git checkout v0.2 >/dev/null 2>&1 || true",
|
||||
"git merge --ff-only origin/v0.2",
|
||||
`test -f ${shellQuote(options.dockerfile)}`,
|
||||
`image=${shellQuote(image)}`,
|
||||
`dockerfile=${shellQuote(options.dockerfile)}`,
|
||||
"echo \"{\\\"phase\\\":\\\"preflight\\\",\\\"image\\\":\\\"$image\\\",\\\"dockerfile\\\":\\\"$dockerfile\\\",\\\"commit\\\":\\\"$(git rev-parse HEAD)\\\"}\"",
|
||||
"export HTTP_PROXY=http://127.0.0.1:10808 HTTPS_PROXY=http://127.0.0.1:10808 http_proxy=http://127.0.0.1:10808 https_proxy=http://127.0.0.1:10808",
|
||||
"export NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal,74.48.78.17,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,10.42.0.0/16,10.43.0.0/16,.svc,.svc.cluster.local,.cluster.local,kubernetes,kubernetes.default,kubernetes.default.svc,127.0.0.1:5000,localhost:5000",
|
||||
"export no_proxy=$NO_PROXY",
|
||||
"docker build --pull --build-arg HTTP_PROXY --build-arg HTTPS_PROXY --build-arg http_proxy --build-arg https_proxy --build-arg NO_PROXY --build-arg no_proxy -f \"$dockerfile\" -t \"$image\" .",
|
||||
"docker run --rm \"$image\" sh -lc 'node --version && npm --version && bun --version && git --version && python3 --version && docker --version && ssh -V'",
|
||||
"docker push \"$image\"",
|
||||
"digest=$(docker image inspect \"$image\" --format '{{index .RepoDigests 0}}' 2>/dev/null || true)",
|
||||
"echo \"{\\\"phase\\\":\\\"published\\\",\\\"image\\\":\\\"$image\\\",\\\"digest\\\":\\\"$digest\\\"}\"",
|
||||
].join("\n");
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
ok: true,
|
||||
command: "hwlab g14 tools-image build --name ci-node-tools",
|
||||
mode: "dry-run",
|
||||
image,
|
||||
workspace: V02_WORKSPACE,
|
||||
dockerfile: options.dockerfile,
|
||||
buildScriptPreview: script.split(/\n/u),
|
||||
next: { build: `bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag ${options.tag} --confirm` },
|
||||
};
|
||||
}
|
||||
const result = g14HostScript(script, options.timeoutSeconds * 1000);
|
||||
return {
|
||||
ok: isCommandSuccess(result),
|
||||
command: "hwlab g14 tools-image build --name ci-node-tools",
|
||||
mode: "confirmed-build",
|
||||
image,
|
||||
workspace: V02_WORKSPACE,
|
||||
dockerfile: options.dockerfile,
|
||||
result,
|
||||
status: runG14ToolsImageStatus({ ...options, action: "status", dryRun: true, confirm: false }),
|
||||
};
|
||||
}
|
||||
|
||||
function runG14ToolsImage(options: G14ToolsImageOptions): Record<string, unknown> {
|
||||
if (options.action === "status") return runG14ToolsImageStatus(options);
|
||||
return runG14ToolsImageBuild(options);
|
||||
}
|
||||
|
||||
function listOpenG14PullRequests(): CommandJsonResult {
|
||||
return cliJson(["gh", "pr", "list", "--repo", HWLAB_REPO, "--state", "open", "--limit", "30", "--json", "number,title,state,url,head,base,draft,headRefName,baseRefName"], 60_000);
|
||||
}
|
||||
@@ -949,16 +1426,25 @@ export function hwlabG14Help(): Record<string, unknown> {
|
||||
"bun scripts/cli.ts hwlab g14 monitor-prs",
|
||||
"bun scripts/cli.ts hwlab g14 monitor-prs --once --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 record-rollout --pr <number> [--source-commit sha]",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane rerun-current --lane v02 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 tools-image status --name ci-node-tools --tag node22-alpine-bun-v1",
|
||||
"bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag node22-alpine-bun-v1 --confirm",
|
||||
"bun scripts/cli.ts job status <jobId> --tail-bytes 30000",
|
||||
],
|
||||
description: "G14 HWLAB PR monitor and DEV rollout command. The public command starts a fire-and-forget job; the worker uses UniDesk gh and ssh routes for every GitHub and k3s operation, then appends the rollout record to the #7-indexed daily brief.",
|
||||
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap helper, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job; control-plane status/apply/rerun-current uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources only.",
|
||||
defaults: {
|
||||
repo: HWLAB_REPO,
|
||||
base: G14_SOURCE_BRANCH,
|
||||
provider: G14_PROVIDER,
|
||||
workspace: G14_WORKSPACE,
|
||||
v02Workspace: V02_WORKSPACE,
|
||||
ciToolsImageRepo: G14_CI_TOOLS_IMAGE_REPO,
|
||||
intervalSeconds: DEFAULT_INTERVAL_SECONDS,
|
||||
devApplication: DEV_APP,
|
||||
v02Application: V02_APP,
|
||||
briefIndexIssue: G14_BRIEF_INDEX_ISSUE,
|
||||
},
|
||||
stateFiles: {
|
||||
@@ -977,8 +1463,16 @@ export async function runHwlabG14Command(_config: Config, args: string[]): Promi
|
||||
const options = parseRecordRolloutOptions(args.slice(1));
|
||||
return appendRolloutBrief(options);
|
||||
}
|
||||
if (action === "control-plane") {
|
||||
const options = parseControlPlaneOptions(args.slice(1));
|
||||
return runV02ControlPlane(options);
|
||||
}
|
||||
if (action === "tools-image") {
|
||||
const options = parseToolsImageOptions(args.slice(1));
|
||||
return runG14ToolsImage(options);
|
||||
}
|
||||
if (action !== "monitor-prs") {
|
||||
return { ok: false, command: `hwlab g14 ${action ?? ""}`.trim(), degradedReason: "unsupported-command", message: "supported commands: hwlab g14 monitor-prs, hwlab g14 record-rollout" };
|
||||
return { ok: false, command: `hwlab g14 ${action ?? ""}`.trim(), degradedReason: "unsupported-command", message: "supported commands: hwlab g14 monitor-prs, hwlab g14 record-rollout, hwlab g14 control-plane, hwlab g14 tools-image" };
|
||||
}
|
||||
const options = parseOptions(args.slice(1));
|
||||
if (options.worker) return runMonitorWorker(options);
|
||||
|
||||
Reference in New Issue
Block a user