Codex
216 lines
16 KiB
TypeScript
216 lines
16 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. help module for scripts/src/hwlab-g14.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-g14.ts:10421-10618 for #903.
|
|
|
|
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand } from "../command";
|
|
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
|
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
|
|
|
import type { G14MonitorOptions } from "./types";
|
|
import { hwlabG14MonitorStateFileName, hwlabG14MonitorStateRole } from "./options";
|
|
import { monitorBaseBranch, record } from "./remote";
|
|
import { legacyG14RetirementBlocksMonitor, legacyG14RetirementStatePath } from "./retirement";
|
|
import { DEFAULT_INTERVAL_SECONDS, DEV_APP, G14_BRIEF_INDEX_ISSUE, G14_CI_TOOLS_IMAGE_REPO, G14_OBSERVABILITY_NAMESPACE, G14_PROMETHEUS_OPERATOR_VERSION, G14_PROMETHEUS_VERSION, G14_PROVIDER, G14_SOURCE_BRANCH, G14_WORKSPACE, HWLAB_REPO, PROD_APP, V02_APP, V02_OBSERVABILITY_EXPECTED_TARGET_COUNT, V02_SOURCE_BRANCH, V02_WORKSPACE } from "./types";
|
|
|
|
export function hwlabG14Help(): Record<string, unknown> {
|
|
return {
|
|
command: "hwlab g14",
|
|
output: "json",
|
|
usage: [
|
|
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v02",
|
|
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v02 --status",
|
|
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v03",
|
|
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v03 --status",
|
|
"bun scripts/cli.ts hwlab g14 retirement status",
|
|
"bun scripts/cli.ts hwlab g14 retirement plan",
|
|
"bun scripts/cli.ts hwlab g14 retirement execute --confirm",
|
|
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v02 --once --dry-run",
|
|
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v03 --once --dry-run",
|
|
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02",
|
|
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --history",
|
|
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --pipeline-run hwlab-v02-ci-poll-<short-sha>",
|
|
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --source-commit <full-sha>",
|
|
"bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --pipeline-run hwlab-v02-ci-poll-<short-sha>",
|
|
"bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --source-commit <full-sha>",
|
|
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --dry-run",
|
|
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm",
|
|
"bun scripts/cli.ts hwlab g14 control-plane refresh --lane v02 --dry-run",
|
|
"bun scripts/cli.ts hwlab g14 control-plane refresh --lane v02 --confirm",
|
|
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm",
|
|
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm --wait",
|
|
"bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03",
|
|
"bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run",
|
|
"bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --confirm",
|
|
"bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --dry-run",
|
|
"bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm",
|
|
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --dry-run",
|
|
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm",
|
|
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm --wait",
|
|
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --dry-run",
|
|
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --confirm",
|
|
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --pipeline-run hwlab-v02-ci-poll-<short-sha> --dry-run",
|
|
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --source-commit <full-sha> --confirm",
|
|
"bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --pipeline-run hwlab-v03-ci-poll-<short-sha> --dry-run",
|
|
"bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --source-commit <full-sha> --confirm",
|
|
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --dry-run",
|
|
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --confirm",
|
|
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --dry-run",
|
|
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --allow-live-db-read --dry-run",
|
|
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --confirm",
|
|
"bun scripts/cli.ts hwlab g14 secret status --lane v02 --name hwlab-v02-openfga",
|
|
"bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-openfga --dry-run",
|
|
"bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-openfga --confirm",
|
|
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga",
|
|
"bun scripts/cli.ts hwlab g14 secret status --lane v02 --name hwlab-v02-master-server-admin-api-key",
|
|
"bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-master-server-admin-api-key --confirm",
|
|
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key",
|
|
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm",
|
|
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db",
|
|
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run",
|
|
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --confirm",
|
|
"bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --dry-run",
|
|
"bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --confirm",
|
|
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider",
|
|
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm",
|
|
"bun scripts/cli.ts hwlab g14 secret delete --lane v02 --name <obsolete-hwlab-v02-secret> --dry-run",
|
|
"bun scripts/cli.ts hwlab g14 secret delete --lane v02 --name <obsolete-hwlab-v02-secret> --confirm",
|
|
"bun scripts/cli.ts hwlab g14 git-mirror status",
|
|
"bun scripts/cli.ts hwlab g14 git-mirror apply --lane v02 --confirm",
|
|
"bun scripts/cli.ts hwlab nodes git-mirror apply --node G14 --lane v03 --confirm",
|
|
"bun scripts/cli.ts hwlab g14 git-mirror sync --confirm",
|
|
"bun scripts/cli.ts hwlab g14 git-mirror flush --confirm",
|
|
"bun scripts/cli.ts hwlab g14 git-mirror sync --confirm --wait",
|
|
"bun scripts/cli.ts hwlab g14 git-mirror flush --confirm --wait",
|
|
"bun scripts/cli.ts hwlab g14 observability status",
|
|
"bun scripts/cli.ts hwlab g14 observability apply --dry-run",
|
|
"bun scripts/cli.ts hwlab g14 observability apply --confirm",
|
|
`bun scripts/cli.ts hwlab g14 observability query --promql 'up{namespace=\"hwlab-v02\"}' --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`,
|
|
`bun scripts/cli.ts hwlab g14 observability query --promql 'hwlab_service_up{namespace=\"hwlab-v02\"}' --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`,
|
|
`bun scripts/cli.ts hwlab g14 observability query --promql 'hwlab_service_health_probe_success{namespace=\"hwlab-v02\"}' --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`,
|
|
"bun scripts/cli.ts hwlab g14 observability targets --lane v02",
|
|
"bun scripts/cli.ts hwlab g14 observability boundary --lane v02",
|
|
"bun scripts/cli.ts hwlab g14 observability closeout --lane v02",
|
|
"bun scripts/cli.ts hwlab g14 tools-image status --name ci-node-tools --tag node22-alpine-bun-v1",
|
|
"bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag node22-alpine-bun-v1 --confirm",
|
|
"bun scripts/cli.ts hwlab g14 upstream-image status --name openfga --tag v1.17.0",
|
|
"bun scripts/cli.ts hwlab g14 upstream-image ensure --name openfga --tag v1.17.0 --dry-run",
|
|
"bun scripts/cli.ts hwlab g14 upstream-image ensure --name openfga --tag v1.17.0 --confirm",
|
|
"bun scripts/cli.ts job status <jobId> --tail-bytes 30000",
|
|
],
|
|
description: "G14 HWLAB PR monitor, legacy DEV/PROD retirement status/plan/execute command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, node-scoped runtime lane v03 control-plane apply/status/refresh/trigger entry, runtime lane SecretRef bootstrap, devops-infra git mirror and observability maintenance, controlled CI tools image build/status entry, and allowlisted upstream image mirroring. The legacy base=G14 monitor is blocked by the retirement contract; the local retirement marker records live execution evidence. `--lane v02` monitors base=v0.2 PRs, waits for GitHub preflight/CI readiness, automatically merges ready PRs without waiting for other active v0.2 PipelineRuns, triggers v0.2 CD with latest-only GitOps writeback, flushes the git mirror when needed, and posts deduplicated PR comments for pending, blocked/conflict, success, superseded, failure, or timeout states. `--lane v03` monitors base=v0.3 PRs through the runtime lane control-plane, waits for GitHub preflight/CI readiness, auto-merges ready non-conflicting PRs, triggers v0.3 CD, verifies PipelineRun/Argo/runtime public probes/Git mirror flush, posts deduplicated PR comments, and creates or updates failure issues for failed checks, conflicts, merge blockers, CD failures, and timeouts. confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane v02 keeps the full closeout/cleanup/runtime-migration verdict path; v03+ is advertised through `hwlab nodes ... --node <node-id> --lane vNN` so node identity remains configuration data instead of a command family. secret status/ensure is the standard runtime lane SecretRef bootstrap path for OpenFGA, cloud-api DB, master admin API key, and code-agent provider refs; it never reads or prints secret values. upstream-image status/ensure only mirrors allowlisted upstream runtime images into the G14 local registry. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob. observability status/apply/query/targets/boundary/closeout owns the shared Prometheus Operator and Prometheus instance in devops-infra, adds bounded PromQL assertions and semantic closeout summaries, while HWLAB lane manifests own only ServiceMonitor and PrometheusRule objects.",
|
|
defaults: {
|
|
repo: HWLAB_REPO,
|
|
legacyBase: G14_SOURCE_BRANCH,
|
|
v02Base: V02_SOURCE_BRANCH,
|
|
runtimeLaneConfig: hwlabRuntimeLaneConfigPath(),
|
|
runtimeLanes: hwlabRuntimeLaneIds(),
|
|
provider: G14_PROVIDER,
|
|
legacyWorkspace: G14_WORKSPACE,
|
|
v02Workspace: V02_WORKSPACE,
|
|
v03Workspace: hwlabRuntimeLaneSpec("v03").workspace,
|
|
ciToolsImageRepo: G14_CI_TOOLS_IMAGE_REPO,
|
|
intervalSeconds: DEFAULT_INTERVAL_SECONDS,
|
|
devApplication: DEV_APP,
|
|
prodApplication: PROD_APP,
|
|
legacyRetirementState: legacyG14RetirementStatePath(),
|
|
v02Application: V02_APP,
|
|
v03Application: hwlabRuntimeLaneSpec("v03").app,
|
|
v03Node: hwlabRuntimeLaneSpec("v03").nodeId,
|
|
v03NetworkProfile: hwlabRuntimeLaneSpec("v03").networkProfileId,
|
|
v03DownloadProfile: hwlabRuntimeLaneSpec("v03").downloadProfileId,
|
|
requiredNoProxy: hwlabRequiredNoProxyEntries(),
|
|
briefIndexIssue: G14_BRIEF_INDEX_ISSUE,
|
|
observabilityNamespace: G14_OBSERVABILITY_NAMESPACE,
|
|
prometheusOperatorVersion: G14_PROMETHEUS_OPERATOR_VERSION,
|
|
prometheusVersion: G14_PROMETHEUS_VERSION,
|
|
},
|
|
stateFiles: {
|
|
monitor: ".state/hwlab-g14/latest-monitor-job.json",
|
|
once: ".state/hwlab-g14/latest-once-job.json",
|
|
dryRun: ".state/hwlab-g14/latest-dry-run-job.json",
|
|
onceDryRun: ".state/hwlab-g14/latest-once-dry-run-job.json",
|
|
v02Monitor: ".state/hwlab-g14/latest-v02-monitor-job.json",
|
|
v02Once: ".state/hwlab-g14/latest-v02-once-job.json",
|
|
v02DryRun: ".state/hwlab-g14/latest-v02-dry-run-job.json",
|
|
v02OnceDryRun: ".state/hwlab-g14/latest-v02-once-dry-run-job.json",
|
|
v02PrCommentSignatures: ".state/hwlab-g14/v02-pr-comment-signatures.json",
|
|
v03Monitor: ".state/hwlab-g14/latest-v03-monitor-job.json",
|
|
v03Once: ".state/hwlab-g14/latest-v03-once-job.json",
|
|
v03DryRun: ".state/hwlab-g14/latest-v03-dry-run-job.json",
|
|
v03OnceDryRun: ".state/hwlab-g14/latest-v03-once-dry-run-job.json",
|
|
v03PrCommentSignatures: ".state/hwlab-g14/v03-pr-comment-signatures.json",
|
|
},
|
|
};
|
|
}
|
|
|
|
export function monitorStatus(options: G14MonitorOptions): Record<string, unknown> {
|
|
const stateDir = rootPath(".state", "hwlab-g14");
|
|
const stateFileName = hwlabG14MonitorStateFileName(options);
|
|
const stateFileRole = hwlabG14MonitorStateRole(options);
|
|
const latestPath = join(stateDir, stateFileName);
|
|
const retirement = options.lane === "g14" ? legacyG14RetirementBlocksMonitor() : null;
|
|
const exists = existsSync(latestPath);
|
|
let latest: Record<string, unknown> | null = null;
|
|
let job: Record<string, unknown> | null = null;
|
|
let statusCommand: string | null = null;
|
|
let degradedReason: string | null = null;
|
|
if (exists) {
|
|
try {
|
|
latest = record(JSON.parse(readFileSync(latestPath, "utf8")) as unknown);
|
|
const jobId = typeof latest.jobId === "string" ? latest.jobId : null;
|
|
if (jobId !== null) {
|
|
statusCommand = `bun scripts/cli.ts job status ${jobId} --tail-bytes 30000`;
|
|
try {
|
|
const current = readJob(jobId);
|
|
job = {
|
|
id: current.id,
|
|
name: current.name,
|
|
status: current.status,
|
|
createdAt: current.createdAt,
|
|
startedAt: current.startedAt,
|
|
finishedAt: current.finishedAt,
|
|
exitCode: current.exitCode,
|
|
runnerPid: current.runnerPid ?? null,
|
|
stdoutFile: current.stdoutFile,
|
|
stderrFile: current.stderrFile,
|
|
note: current.note,
|
|
};
|
|
} catch (error) {
|
|
degradedReason = error instanceof Error ? error.message : String(error);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
degradedReason = `failed to parse ${latestPath}: ${error instanceof Error ? error.message : String(error)}`;
|
|
}
|
|
}
|
|
return {
|
|
ok: degradedReason === null,
|
|
command: "hwlab g14 monitor-prs --status",
|
|
lane: options.lane,
|
|
baseBranch: monitorBaseBranch(options.lane),
|
|
mode: "status",
|
|
latestPath,
|
|
stateFileName,
|
|
stateFileRole,
|
|
exists,
|
|
latest,
|
|
job,
|
|
statusCommand,
|
|
degradedReason,
|
|
retirement,
|
|
next: retirement !== null ? {
|
|
retirementStatus: "bun scripts/cli.ts hwlab g14 retirement status",
|
|
v02Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v02",
|
|
v03Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v03",
|
|
} : statusCommand === null ? {
|
|
start: `bun scripts/cli.ts hwlab g14 monitor-prs --lane ${options.lane}`,
|
|
} : {
|
|
status: statusCommand,
|
|
},
|
|
};
|
|
}
|