refactor: add node-scoped HWLAB lane CLI

This commit is contained in:
Codex
2026-06-08 14:18:58 +00:00
parent b3476b49f1
commit 8f51bbca91
8 changed files with 508 additions and 116 deletions
+2 -1
View File
@@ -59,7 +59,8 @@ export function rootHelp(): unknown {
{ command: "auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run", description: "Inspect the P0 Rust auth broker and CLI adapter contract without reading token values, writing GitHub, or starting services." },
{ command: "gh preflight|auth|issue|pr", description: "Run safe GitHub issue and PR CRUD/lifecycle operations through REST with body-file update replace/append, comment delete, token diagnostics, PR closeout preflight, hard delete unsupported, and guarded PR merge." },
{ command: "commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr", description: "Host Codex commander skeleton contract, no-daemon smoke plan, dry-run approval preview, and advisory GPT-5.5 PR prompt boundary lint without live bridges, message sends, or submit gating." },
{ command: "hwlab nodes secret status|ensure --node G14 --lane v03 | hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|trigger-current|runtime-migration|cleanup-runs|cleanup-released-pvs | hwlab g14 git-mirror status|apply|sync|flush | hwlab g14 tools-image status|build", description: "Manage HWLAB node/lane runtime prerequisites, start the legacy G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane, manual PipelineRun trigger, runtime migration, CI workspace retention, manual devops-infra git mirror/relay maintenance, or fixed HWLAB CI tools image actions; long confirmed trigger/sync/flush actions return async jobs by default." },
{ command: "hwlab nodes control-plane|git-mirror|secret --node G14 --lane v03", description: "Manage HWLAB node/lane runtime prerequisites for v0.3+ with the node identity passed as data instead of a command family." },
{ command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|trigger-current|runtime-migration|cleanup-runs|cleanup-released-pvs | hwlab g14 git-mirror status|apply|sync|flush | hwlab g14 tools-image status|build", description: "Start the legacy G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane, manual PipelineRun trigger, runtime migration, CI workspace retention, manual devops-infra git mirror/relay maintenance, or fixed HWLAB CI tools image actions; long confirmed trigger/sync/flush actions return async jobs by default." },
{ command: "agentrun v01 control-plane status|trigger-current|refresh|cleanup-runs|cleanup-released-pvs", description: "Run bounded AgentRun v0.1 Tekton/Argo status, manual PipelineRun trigger, Argo refresh, and completed CI workspace retention through UniDesk G14 routes." },
{ command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Legacy D601 HWLAB DEV CD wrapper kept for explicit old-path diagnostics; current HWLAB rollout uses G14 GitOps." },
{ command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." },
+311 -69
View File
@@ -4,7 +4,7 @@ import { createHash, randomBytes } from "node:crypto";
import { repoRoot, rootPath, type Config } from "./config";
import { runCommand } from "./command";
import { readJob, startJob } from "./jobs";
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-g14-lanes";
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
const HWLAB_REPO = "pikasTech/HWLAB";
const G14_SOURCE_BRANCH = "G14";
@@ -24,7 +24,6 @@ const V02_POLLER = "hwlab-v02-branch-poller";
const V02_RECONCILER = "hwlab-v02-control-plane-reconciler";
const V02_PIPELINERUN_PREFIX = V02_LANE_SPEC.pipelineRunPrefix;
const V02_CONTROL_PLANE_FIELD_MANAGER = V02_LANE_SPEC.controlPlaneFieldManager;
const V02_SECRET_FIELD_MANAGER = "unidesk-hwlab-v02-secret";
const V02_GIT_URL = V02_LANE_SPEC.gitUrl;
const V02_GIT_READ_URL = V02_LANE_SPEC.gitReadUrl;
const V02_GIT_WRITE_URL = V02_LANE_SPEC.gitWriteUrl;
@@ -123,7 +122,7 @@ interface G14RecordRolloutOptions {
}
interface G14ControlPlaneOptions {
action: "status" | "closeout" | "apply" | "trigger-current" | "cleanup-runs" | "cleanup-released-pvs" | "runtime-migration";
action: "status" | "closeout" | "apply" | "trigger-current" | "refresh" | "cleanup-runs" | "cleanup-released-pvs" | "runtime-migration";
lane: HwlabRuntimeLane | "g14" | "all";
dryRun: boolean;
confirm: boolean;
@@ -188,7 +187,7 @@ interface G14ObservabilityOptions {
interface G14SecretOptions {
action: "status" | "ensure" | "delete";
lane: "v02";
lane: HwlabRuntimeLane;
dryRun: boolean;
confirm: boolean;
name: string;
@@ -357,11 +356,12 @@ function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions {
actionRaw !== "closeout" &&
actionRaw !== "apply" &&
actionRaw !== "trigger-current" &&
actionRaw !== "refresh" &&
actionRaw !== "cleanup-runs" &&
actionRaw !== "cleanup-released-pvs" &&
actionRaw !== "runtime-migration"
) {
throw new Error("control-plane usage: status|apply|trigger-current --lane v02|v03 | closeout|runtime-migration --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]");
throw new Error("control-plane usage: status|apply|trigger-current|refresh --lane v02|v03 | closeout|runtime-migration --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]");
}
const laneRaw = optionValue(args, "--lane") ?? (actionRaw === "cleanup-released-pvs" ? "all" : "v02");
let lane: G14ControlPlaneOptions["lane"];
@@ -538,16 +538,22 @@ function parseObservabilityOptions(args: string[]): G14ObservabilityOptions {
function parseSecretOptions(args: string[]): G14SecretOptions {
const [actionRaw] = args;
if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "delete") {
throw new Error("secret usage: status|ensure --lane v02 --name hwlab-v02-openfga|hwlab-v02-master-server-admin-api-key [--dry-run|--confirm] | delete --lane v02 --name <obsolete-secret> [--dry-run|--confirm]");
throw new Error("secret usage: status|ensure --lane v02|v03 --name hwlab-v0x-openfga|hwlab-v0x-master-server-admin-api-key [--dry-run|--confirm] | delete --lane v02 --name <obsolete-secret> [--dry-run|--confirm]");
}
const lane = optionValue(args, "--lane") ?? "v02";
if (lane !== "v02") throw new Error("secret currently supports --lane v02");
const name = optionValue(args, "--name") ?? V02_OPENFGA_SECRET;
const laneRaw = optionValue(args, "--lane") ?? "v02";
if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`secret --lane must be one of ${hwlabRuntimeLaneIds().join(", ")}`);
const lane = laneRaw;
const spec = hwlabRuntimeLaneSpec(lane);
const openFgaSecret = runtimeLaneOpenFgaSecretName(spec);
const masterAdminSecret = runtimeLaneMasterAdminApiKeySecretName(spec);
const postgresSecret = runtimeLanePostgresSecretName(spec);
const name = optionValue(args, "--name") ?? openFgaSecret;
const key = optionValue(args, "--key");
const confirm = args.includes("--confirm");
const explicitDryRun = args.includes("--dry-run");
if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run");
if (actionRaw === "delete") {
if (lane !== "v02") throw new Error("secret delete is currently limited to --lane v02 obsolete cleanup");
if (key !== undefined) throw new Error("secret delete does not accept --key; it deletes a whole obsolete Secret object");
if (!/^hwlab-v02-[a-z0-9-]+$/u.test(name)) throw new Error("secret delete requires a hwlab-v02-* Secret name");
if (name === V02_OPENFGA_SECRET || name === V02_MASTER_ADMIN_API_KEY_SECRET || name === "hwlab-v02-postgres") throw new Error(`secret delete refuses required v0.2 Secret ${name}`);
@@ -561,8 +567,8 @@ function parseSecretOptions(args: string[]): G14SecretOptions {
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
};
}
if (name === V02_MASTER_ADMIN_API_KEY_SECRET) {
if (key !== undefined && key !== V02_MASTER_ADMIN_API_KEY_SECRET_KEY) throw new Error(`secret ${V02_MASTER_ADMIN_API_KEY_SECRET} supports only key ${V02_MASTER_ADMIN_API_KEY_SECRET_KEY}`);
if (name === masterAdminSecret) {
if (key !== undefined && key !== V02_MASTER_ADMIN_API_KEY_SECRET_KEY) throw new Error(`secret ${masterAdminSecret} supports only key ${V02_MASTER_ADMIN_API_KEY_SECRET_KEY}`);
return {
action: actionRaw,
lane,
@@ -574,12 +580,13 @@ function parseSecretOptions(args: string[]): G14SecretOptions {
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
};
}
if (name !== V02_OPENFGA_SECRET) {
throw new Error(`secret status/ensure currently supports only --name ${V02_OPENFGA_SECRET} or ${V02_MASTER_ADMIN_API_KEY_SECRET}; use secret delete for obsolete Secret objects`);
if (name !== openFgaSecret) {
throw new Error(`secret status/ensure currently supports only --name ${openFgaSecret} or ${masterAdminSecret} for --lane ${lane}; use secret delete for obsolete v02 Secret objects`);
}
if (key !== undefined && key !== V02_OPENFGA_AUTHN_SECRET_KEY && key !== V02_OPENFGA_DATASTORE_URI_SECRET_KEY && key !== V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY) {
throw new Error(`secret ${V02_OPENFGA_SECRET} supports keys ${V02_OPENFGA_AUTHN_SECRET_KEY}, ${V02_OPENFGA_DATASTORE_URI_SECRET_KEY}, and ${V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY}`);
throw new Error(`secret ${openFgaSecret} supports keys ${V02_OPENFGA_AUTHN_SECRET_KEY}, ${V02_OPENFGA_DATASTORE_URI_SECRET_KEY}, and ${V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY}`);
}
if (lane !== "v02" && key === V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY) throw new Error(`${openFgaSecret}/${V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY} is derived from ${postgresSecret}/POSTGRES_PASSWORD`);
return {
action: actionRaw,
lane,
@@ -717,17 +724,43 @@ function hwlabApiKeyPrefix(value: string): string {
return `hwl_live_${segment}`.slice(0, 24);
}
function localMasterAdminApiKeyStatus(): Record<string, unknown> {
if (!existsSync(V02_MASTER_ADMIN_API_KEY_LOCAL_ENV)) {
return { exists: false, path: V02_MASTER_ADMIN_API_KEY_LOCAL_ENV, mode: null, valueBytes: 0, keyPrefix: null };
function runtimeLaneSecretFieldManager(spec: HwlabRuntimeLaneSpec): string {
return `unidesk-hwlab-${spec.lane}-secret`;
}
function runtimeLanePostgresSecretName(spec: HwlabRuntimeLaneSpec): string {
return `${spec.runtimeNamespace}-postgres`;
}
function runtimeLanePrimaryDbName(spec: HwlabRuntimeLaneSpec): string {
return `hwlab_${spec.lane}`;
}
function runtimeLaneOpenFgaSecretName(spec: HwlabRuntimeLaneSpec): string {
return `hwlab-${spec.lane}-openfga`;
}
function runtimeLaneMasterAdminApiKeySecretName(spec: HwlabRuntimeLaneSpec): string {
return `hwlab-${spec.lane}-master-server-admin-api-key`;
}
function runtimeLaneMasterAdminApiKeyLocalEnv(spec: HwlabRuntimeLaneSpec): string {
if (spec.lane === "v02") return V02_MASTER_ADMIN_API_KEY_LOCAL_ENV;
return `/root/.config/hwlab-${spec.lane}/master-server-admin-api-key.env`;
}
function localMasterAdminApiKeyStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
const envPath = runtimeLaneMasterAdminApiKeyLocalEnv(spec);
if (!existsSync(envPath)) {
return { exists: false, path: envPath, mode: null, valueBytes: 0, keyPrefix: null };
}
const stat = statSync(V02_MASTER_ADMIN_API_KEY_LOCAL_ENV);
const content = readFileSync(V02_MASTER_ADMIN_API_KEY_LOCAL_ENV, "utf8");
const stat = statSync(envPath);
const content = readFileSync(envPath, "utf8");
const match = content.match(/^HWLAB_API_KEY=(.+)$/mu);
const value = match?.[1]?.trim() ?? "";
return {
exists: true,
path: V02_MASTER_ADMIN_API_KEY_LOCAL_ENV,
path: envPath,
mode: `0${(stat.mode & 0o777).toString(8)}`,
valueBytes: Buffer.byteLength(value, "utf8"),
keyPrefix: value ? hwlabApiKeyPrefix(value) : null,
@@ -735,21 +768,22 @@ function localMasterAdminApiKeyStatus(): Record<string, unknown> {
};
}
function readOrCreateLocalMasterAdminApiKey(dryRun: boolean): { key: string | null; created: boolean; status: Record<string, unknown> } {
const existing = localMasterAdminApiKeyStatus();
function readOrCreateLocalMasterAdminApiKey(spec: HwlabRuntimeLaneSpec, dryRun: boolean): { key: string | null; created: boolean; status: Record<string, unknown> } {
const envPath = runtimeLaneMasterAdminApiKeyLocalEnv(spec);
const existing = localMasterAdminApiKeyStatus(spec);
if (existing.exists === true) {
const content = readFileSync(V02_MASTER_ADMIN_API_KEY_LOCAL_ENV, "utf8");
const content = readFileSync(envPath, "utf8");
const match = content.match(/^HWLAB_API_KEY=(.+)$/mu);
const key = match?.[1]?.trim() ?? "";
if (!key.startsWith("hwl_live_")) throw new Error(`${V02_MASTER_ADMIN_API_KEY_LOCAL_ENV} exists but does not contain a hwl_live_ HWLAB_API_KEY`);
if (!dryRun && existing.mode !== "0600") chmodSync(V02_MASTER_ADMIN_API_KEY_LOCAL_ENV, 0o600);
return { key, created: false, status: localMasterAdminApiKeyStatus() };
if (!key.startsWith("hwl_live_")) throw new Error(`${envPath} exists but does not contain a hwl_live_ HWLAB_API_KEY`);
if (!dryRun && existing.mode !== "0600") chmodSync(envPath, 0o600);
return { key, created: false, status: localMasterAdminApiKeyStatus(spec) };
}
if (dryRun) return { key: null, created: false, status: existing };
const key = generateHwlabApiKey();
mkdirSync(dirname(V02_MASTER_ADMIN_API_KEY_LOCAL_ENV), { recursive: true, mode: 0o700 });
writeFileSync(V02_MASTER_ADMIN_API_KEY_LOCAL_ENV, `# HWLAB v0.2 master server admin API key; do not commit or print.\nHWLAB_API_KEY=${key}\n`, { mode: 0o600 });
return { key, created: true, status: localMasterAdminApiKeyStatus() };
mkdirSync(dirname(envPath), { recursive: true, mode: 0o700 });
writeFileSync(envPath, `# HWLAB ${spec.version} master server admin API key; do not commit or print.\nHWLAB_API_KEY=${key}\n`, { mode: 0o600 });
return { key, created: true, status: localMasterAdminApiKeyStatus(spec) };
}
function redactLargePayloads(value: string): string {
@@ -2877,10 +2911,18 @@ function runtimeLaneControlPlaneRenderScript(spec: HwlabRuntimeLaneSpec, sourceC
" exit 42",
" fi",
"fi",
"if [ -f scripts/run-bun.mjs ]; then",
` node scripts/run-bun.mjs scripts/g14-gitops-render.mjs --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
"if [ -f scripts/gitops-render.mjs ]; then",
" render_script=scripts/gitops-render.mjs",
"elif [ \"$render_lane\" = v02 ] && [ -f scripts/g14-gitops-render.mjs ]; then",
" render_script=scripts/g14-gitops-render.mjs",
"else",
` node scripts/g14-gitops-render.mjs --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
` echo "${spec.version} control-plane render script missing: scripts/gitops-render.mjs" >&2`,
" exit 43",
"fi",
"if [ -f scripts/run-bun.mjs ]; then",
` node scripts/run-bun.mjs "$render_script" --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
"else",
` node "$render_script" --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
"fi",
].join("\n");
}
@@ -3664,11 +3706,158 @@ function runtimeLaneControlPlaneStatus(spec: HwlabRuntimeLaneSpec, target: V02Co
},
next: {
apply: `bun scripts/cli.ts hwlab g14 control-plane apply --lane ${spec.lane} --confirm`,
refresh: `bun scripts/cli.ts hwlab g14 control-plane refresh --lane ${spec.lane} --confirm`,
triggerCurrent: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm`,
},
};
}
function runtimeLaneArgoRefreshScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string {
const operationPatch = JSON.stringify({ operation: null });
return [
"set +e",
`app=${shellQuote(spec.app)}`,
`argo_namespace=${shellQuote(ARGO_NAMESPACE)}`,
`runtime_namespace=${shellQuote(spec.runtimeNamespace)}`,
`lane=${shellQuote(spec.lane)}`,
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
`operation_patch=${shellQuote(operationPatch)}`,
"app_jsonpath() { kubectl -n \"$argo_namespace\" get application \"$app\" -o \"jsonpath=$1\" 2>/dev/null || true; }",
"job_jsonpath() { kubectl -n \"$runtime_namespace\" get job hwlab-openfga-migrate -o \"jsonpath=$1\" 2>/dev/null || true; }",
"before_operation_revision=$(app_jsonpath '{.operation.sync.revision}')",
"before_operation_phase=$(app_jsonpath '{.status.operationState.phase}')",
"before_operation_message=$(app_jsonpath '{.status.operationState.message}')",
"before_sync_revision=$(app_jsonpath '{.status.sync.revision}')",
"before_sync_status=$(app_jsonpath '{.status.sync.status}')",
"before_health=$(app_jsonpath '{.status.health.status}')",
"before_resource_version=$(app_jsonpath '{.metadata.resourceVersion}')",
"before_hook_job=$(kubectl -n \"$runtime_namespace\" get job hwlab-openfga-migrate -o name 2>/dev/null || true)",
"before_hook_source=$(job_jsonpath '{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}')",
"before_hook_phase=$(job_jsonpath '{.status.conditions[0].type}')",
"before_hook_active=$(job_jsonpath '{.status.active}')",
"before_operation_present=$([ -n \"$before_operation_revision$before_operation_phase$before_operation_message\" ] && printf yes || printf no)",
"patch_exit=",
"patch_output=skipped-no-live-operation",
"annotate_exit=",
"annotate_output=",
"if [ \"$before_operation_present\" = yes ]; then",
" if [ \"$dry_run\" = true ]; then",
" patch_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type=merge -p \"$operation_patch\" --dry-run=server -o name 2>&1)",
" else",
" patch_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type=merge -p \"$operation_patch\" -o name 2>&1)",
" fi",
" patch_exit=$?",
"fi",
"if [ \"$dry_run\" = true ]; then",
" annotate_output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite --dry-run=server -o name 2>&1)",
"else",
" annotate_output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite -o name 2>&1)",
"fi",
"annotate_exit=$?",
"if [ \"$dry_run\" != true ]; then sleep 2; fi",
"after_operation_revision=$(app_jsonpath '{.operation.sync.revision}')",
"after_operation_phase=$(app_jsonpath '{.status.operationState.phase}')",
"after_operation_message=$(app_jsonpath '{.status.operationState.message}')",
"after_sync_revision=$(app_jsonpath '{.status.sync.revision}')",
"after_sync_status=$(app_jsonpath '{.status.sync.status}')",
"after_health=$(app_jsonpath '{.status.health.status}')",
"after_resource_version=$(app_jsonpath '{.metadata.resourceVersion}')",
"after_hook_job=$(kubectl -n \"$runtime_namespace\" get job hwlab-openfga-migrate -o name 2>/dev/null || true)",
"after_hook_source=$(job_jsonpath '{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}')",
"after_hook_phase=$(job_jsonpath '{.status.conditions[0].type}')",
"after_hook_active=$(job_jsonpath '{.status.active}')",
"after_operation_present=$([ -n \"$after_operation_revision$after_operation_phase$after_operation_message\" ] && printf yes || printf no)",
"printf 'app\\t%s\\n' \"$app\"",
"printf 'lane\\t%s\\n' \"$lane\"",
"printf 'runtimeNamespace\\t%s\\n' \"$runtime_namespace\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'beforeOperationPresent\\t%s\\n' \"$before_operation_present\"",
"printf 'beforeOperationRevision\\t%s\\n' \"$before_operation_revision\"",
"printf 'beforeOperationPhase\\t%s\\n' \"$before_operation_phase\"",
"printf 'beforeOperationMessage\\t%s\\n' \"$before_operation_message\"",
"printf 'beforeSyncRevision\\t%s\\n' \"$before_sync_revision\"",
"printf 'beforeSyncStatus\\t%s\\n' \"$before_sync_status\"",
"printf 'beforeHealth\\t%s\\n' \"$before_health\"",
"printf 'beforeResourceVersion\\t%s\\n' \"$before_resource_version\"",
"printf 'beforeHookJob\\t%s\\n' \"$before_hook_job\"",
"printf 'beforeHookSourceCommit\\t%s\\n' \"$before_hook_source\"",
"printf 'beforeHookPhase\\t%s\\n' \"$before_hook_phase\"",
"printf 'beforeHookActive\\t%s\\n' \"$before_hook_active\"",
"printf 'patchExitCode\\t%s\\n' \"$patch_exit\"",
"printf 'patchOutput\\t%s\\n' \"$patch_output\"",
"printf 'annotateExitCode\\t%s\\n' \"$annotate_exit\"",
"printf 'annotateOutput\\t%s\\n' \"$annotate_output\"",
"printf 'afterOperationPresent\\t%s\\n' \"$after_operation_present\"",
"printf 'afterOperationRevision\\t%s\\n' \"$after_operation_revision\"",
"printf 'afterOperationPhase\\t%s\\n' \"$after_operation_phase\"",
"printf 'afterOperationMessage\\t%s\\n' \"$after_operation_message\"",
"printf 'afterSyncRevision\\t%s\\n' \"$after_sync_revision\"",
"printf 'afterSyncStatus\\t%s\\n' \"$after_sync_status\"",
"printf 'afterHealth\\t%s\\n' \"$after_health\"",
"printf 'afterResourceVersion\\t%s\\n' \"$after_resource_version\"",
"printf 'afterHookJob\\t%s\\n' \"$after_hook_job\"",
"printf 'afterHookSourceCommit\\t%s\\n' \"$after_hook_source\"",
"printf 'afterHookPhase\\t%s\\n' \"$after_hook_phase\"",
"printf 'afterHookActive\\t%s\\n' \"$after_hook_active\"",
"if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then exit \"$patch_exit\"; fi",
"if [ -n \"$annotate_exit\" ] && [ \"$annotate_exit\" != 0 ]; then exit \"$annotate_exit\"; fi",
].join("\n");
}
function refreshRuntimeLaneArgoApplication(spec: HwlabRuntimeLaneSpec, options: G14ControlPlaneOptions): Record<string, unknown> {
const result = g14K3s(["script", "--", runtimeLaneArgoRefreshScript(spec, options.dryRun)], options.timeoutSeconds * 1000);
const fields = keyValueLinesFromText(statusText(result));
const ok = isCommandSuccess(result);
return {
ok,
command: `hwlab g14 control-plane refresh --lane ${spec.lane}`,
lane: spec.lane,
mode: options.dryRun ? "dry-run" : "confirmed-refresh",
argoApplication: spec.app,
runtimeNamespace: spec.runtimeNamespace,
dryRun: options.dryRun,
action: {
terminatedOperation: fields.beforeOperationPresent === "yes",
hardRefreshRequested: numericField(fields.annotateExitCode) === 0,
mutation: !options.dryRun && ok,
},
before: {
operationPresent: fields.beforeOperationPresent === "yes",
operationRevision: fields.beforeOperationRevision || null,
operationPhase: fields.beforeOperationPhase || null,
operationMessage: fields.beforeOperationMessage || null,
syncRevision: fields.beforeSyncRevision || null,
syncStatus: fields.beforeSyncStatus || null,
health: fields.beforeHealth || null,
resourceVersion: fields.beforeResourceVersion || null,
hookJob: fields.beforeHookJob || null,
hookSourceCommit: fields.beforeHookSourceCommit || null,
hookPhase: fields.beforeHookPhase || null,
hookActive: fields.beforeHookActive || null,
},
after: {
operationPresent: fields.afterOperationPresent === "yes",
operationRevision: fields.afterOperationRevision || null,
operationPhase: fields.afterOperationPhase || null,
operationMessage: fields.afterOperationMessage || null,
syncRevision: fields.afterSyncRevision || null,
syncStatus: fields.afterSyncStatus || null,
health: fields.afterHealth || null,
resourceVersion: fields.afterResourceVersion || null,
hookJob: fields.afterHookJob || null,
hookSourceCommit: fields.afterHookSourceCommit || null,
hookPhase: fields.afterHookPhase || null,
hookActive: fields.afterHookActive || null,
},
patchExitCode: numericField(fields.patchExitCode),
annotateExitCode: numericField(fields.annotateExitCode),
result: compactCommandResult(result),
next: {
status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${spec.lane}`,
},
};
}
function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14ControlPlaneOptions): Record<string, unknown> {
if (options.action === "closeout" || options.action === "runtime-migration" || options.action === "cleanup-runs" || options.action === "cleanup-released-pvs") {
return {
@@ -3676,7 +3865,7 @@ function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14Cont
command: `hwlab g14 control-plane ${options.action} --lane ${spec.lane}`,
lane: spec.lane,
degradedReason: "unsupported-runtime-lane-action",
message: `${options.action} is still v0.2-specific; v0.3+ currently supports status/apply/trigger-current`,
message: `${options.action} is still v0.2-specific; v0.3+ currently supports status/apply/trigger-current/refresh`,
};
}
if (options.action === "status" && options.pipelineRun !== undefined) {
@@ -3722,6 +3911,9 @@ function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14Cont
if (options.action === "status") {
return runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "latest-source-head", includeHistory: options.history });
}
if (options.action === "refresh") {
return refreshRuntimeLaneArgoApplication(spec, options);
}
if (options.action === "apply") {
const render = runRuntimeLaneRenderToTemp(spec, sourceCommit);
if (!isCommandSuccess(render.result)) {
@@ -3937,6 +4129,7 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
}
if (options.action === "runtime-migration") return runV02RuntimeMigration(options, sourceCommit);
if (options.action === "status") return v02ControlPlaneStatus({ sourceCommit, mode: "latest-source-head", includeHistory: options.history });
if (options.action === "refresh") return refreshRuntimeLaneArgoApplication(V02_LANE_SPEC, options);
if (options.action === "apply") {
const render = runV02RenderToTemp(sourceCommit);
if (!isCommandSuccess(render.result)) {
@@ -4172,10 +4365,11 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
}
function v02SecretScript(options: G14SecretOptions): string {
const spec = hwlabRuntimeLaneSpec(options.lane);
if (options.preset === "generic-delete") return v02DeleteSecretScript(options);
if (options.preset === "master-server-admin-api-key") return v02MasterAdminApiKeySecretScript(options);
if (options.preset === "master-server-admin-api-key") return v02MasterAdminApiKeySecretScript(options, spec);
return [
v02OpenFgaSecretScript(options),
v02OpenFgaSecretScript(options, spec),
].join("\n");
}
@@ -4216,36 +4410,56 @@ function v02DeleteSecretScript(options: G14SecretOptions): string {
].join("\n");
}
function v02OpenFgaSecretScript(options: G14SecretOptions): string {
function v02OpenFgaSecretScript(options: G14SecretOptions, spec: HwlabRuntimeLaneSpec): string {
const selectedKey = options.key ?? "";
const namespace = spec.runtimeNamespace;
const openFgaSecret = runtimeLaneOpenFgaSecretName(spec);
const postgresSecret = runtimeLanePostgresSecretName(spec);
const primaryDbName = runtimeLanePrimaryDbName(spec);
const dedicatedDb = spec.lane === "v02";
const dbMode = dedicatedDb ? "dedicated" : "primary";
const dbName = dedicatedDb ? V02_OPENFGA_DB_NAME : primaryDbName;
const dbUser = dedicatedDb ? V02_OPENFGA_DB_USER : primaryDbName;
const dbHost = `${postgresSecret}.${namespace}.svc.cluster.local`;
const fieldManager = runtimeLaneSecretFieldManager(spec);
return [
"set +e",
`namespace=${shellQuote(V02_RUNTIME_NAMESPACE)}`,
`name=${shellQuote(V02_OPENFGA_SECRET)}`,
`namespace=${shellQuote(namespace)}`,
`name=${shellQuote(openFgaSecret)}`,
`selected_key=${shellQuote(selectedKey)}`,
`authn_key=${shellQuote(V02_OPENFGA_AUTHN_SECRET_KEY)}`,
`datastore_uri_key=${shellQuote(V02_OPENFGA_DATASTORE_URI_SECRET_KEY)}`,
`postgres_password_key=${shellQuote(V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY)}`,
`db_name=${shellQuote(V02_OPENFGA_DB_NAME)}`,
`db_user=${shellQuote(V02_OPENFGA_DB_USER)}`,
"db_host=hwlab-v02-postgres.hwlab-v02.svc.cluster.local",
`db_mode=${shellQuote(dbMode)}`,
`db_name=${shellQuote(dbName)}`,
`db_user=${shellQuote(dbUser)}`,
`postgres_secret=${shellQuote(postgresSecret)}`,
`postgres_statefulset=${shellQuote(postgresSecret)}`,
`postgres_admin_user=${shellQuote(primaryDbName)}`,
`db_host=${shellQuote(dbHost)}`,
`action_request=${shellQuote(options.action)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
`field_manager=${shellQuote(V02_SECRET_FIELD_MANAGER)}`,
`field_manager=${shellQuote(fieldManager)}`,
"preset=openfga",
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }",
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }",
"decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }",
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
"psql_scalar() { kubectl -n \"$namespace\" exec statefulset/hwlab-v02-postgres -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U hwlab_v02 -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }",
"psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }",
"probe_db() {",
" if [ \"$db_mode\" = primary ]; then",
" role_result=primary",
" database_result=primary",
" probe_exit=not-required",
" return",
" fi",
" role_result=unknown",
" database_result=unknown",
" probe_exit=missing-postgres-admin-secret",
" if [ -n \"$postgres_admin_password\" ]; then",
` role_result=$(psql_scalar "select exists(select 1 from pg_roles where rolname='${V02_OPENFGA_DB_USER}');")`,
" role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")",
" role_exit=$?",
` database_result=$(psql_scalar "select exists(select 1 from pg_database where datname='${V02_OPENFGA_DB_NAME}');")`,
" database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")",
" database_exit=$?",
" if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi",
" fi",
@@ -4263,7 +4477,7 @@ function v02OpenFgaSecretScript(options: G14SecretOptions): string {
"authn_value=$(decoded_value \"$before_authn_b64\")",
"datastore_uri=$(decoded_value \"$before_uri_b64\")",
"pg_password=$(decoded_value \"$before_pg_password_b64\")",
"postgres_admin_b64=$(kubectl -n \"$namespace\" get secret hwlab-v02-postgres -o 'go-template={{ index .data \"POSTGRES_PASSWORD\" }}' 2>/dev/null || true)",
"postgres_admin_b64=$(kubectl -n \"$namespace\" get secret \"$postgres_secret\" -o 'go-template={{ index .data \"POSTGRES_PASSWORD\" }}' 2>/dev/null || true)",
"postgres_admin_present=$([ -n \"$postgres_admin_b64\" ] && printf yes || printf no)",
"postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")",
"probe_db",
@@ -4280,8 +4494,10 @@ function v02OpenFgaSecretScript(options: G14SecretOptions): string {
" [ \"$before_uri_present\" = yes ] && [ \"$before_uri_bytes\" -gt 0 ] || missing_secret=true",
" [ \"$before_pg_password_present\" = yes ] && [ \"$before_pg_password_bytes\" -gt 0 ] || missing_secret=true",
" missing_db=false",
" [ \"$db_role_exists_before\" = t ] || missing_db=true",
" [ \"$db_database_exists_before\" = t ] || missing_db=true",
" if [ \"$db_mode\" = dedicated ]; then",
" [ \"$db_role_exists_before\" = t ] || missing_db=true",
" [ \"$db_database_exists_before\" = t ] || missing_db=true",
" fi",
" if [ \"$dry_run\" = true ]; then",
" if [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi",
" elif ! command -v openssl >/dev/null 2>&1; then",
@@ -4292,12 +4508,20 @@ function v02OpenFgaSecretScript(options: G14SecretOptions): string {
" apply_exit=44",
" else",
" [ -n \"$authn_value\" ] || authn_value=$(openssl rand -base64 48)",
" [ -n \"$pg_password\" ] || pg_password=$(openssl rand -hex 24)",
" if [ \"$db_mode\" = primary ]; then",
" [ -n \"$pg_password\" ] || pg_password=\"$postgres_admin_password\"",
" else",
" [ -n \"$pg_password\" ] || pg_password=$(openssl rand -hex 24)",
" fi",
" [ -n \"$datastore_uri\" ] || datastore_uri=\"postgres://$db_user:$pg_password@$db_host:5432/$db_name?sslmode=disable\"",
" kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$authn_key=$authn_value\" --from-literal=\"$datastore_uri_key=$datastore_uri\" --from-literal=\"$postgres_password_key=$pg_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
" apply_exit=$?",
" if [ \"$apply_exit\" -eq 0 ]; then",
" kubectl -n \"$namespace\" exec -i statefulset/hwlab-v02-postgres -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U hwlab_v02 -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$pg_password\" >/tmp/hwlab-openfga-psql.out 2>/tmp/hwlab-openfga-psql.err <<'SQL'",
" if [ \"$db_mode\" = primary ]; then",
" action=ensured",
" mutation=true",
" else",
" kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$pg_password\" >/tmp/hwlab-openfga-psql.out 2>/tmp/hwlab-openfga-psql.err <<'SQL'",
"SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')",
"WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')",
"\\gexec",
@@ -4307,8 +4531,9 @@ function v02OpenFgaSecretScript(options: G14SecretOptions): string {
"\\gexec",
"ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";",
"SQL",
" db_ensure_exit=$?",
" if [ \"$db_ensure_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=db-ensure-failed; fi",
" db_ensure_exit=$?",
" if [ \"$db_ensure_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=db-ensure-failed; fi",
" fi",
" else",
" action=apply-failed",
" fi",
@@ -4335,6 +4560,7 @@ function v02OpenFgaSecretScript(options: G14SecretOptions): string {
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'dbMode\\t%s\\n' \"$db_mode\"",
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
"printf 'beforeAuthnPresent\\t%s\\n' \"$before_authn_present\"",
"printf 'beforeAuthnBytes\\t%s\\n' \"$before_authn_bytes\"",
@@ -4350,6 +4576,7 @@ function v02OpenFgaSecretScript(options: G14SecretOptions): string {
"printf 'afterPostgresPasswordPresent\\t%s\\n' \"$after_pg_password_present\"",
"printf 'afterPostgresPasswordBytes\\t%s\\n' \"$after_pg_password_bytes\"",
"printf 'postgresAdminSecretPresent\\t%s\\n' \"$postgres_admin_present\"",
"printf 'postgresSecret\\t%s\\n' \"$postgres_secret\"",
"printf 'dbName\\t%s\\n' \"$db_name\"",
"printf 'dbUser\\t%s\\n' \"$db_user\"",
"printf 'dbRoleExistsBefore\\t%s\\n' \"$db_role_exists_before\"",
@@ -4369,15 +4596,18 @@ function v02OpenFgaSecretScript(options: G14SecretOptions): string {
].join("\n");
}
function v02MasterAdminApiKeySecretScript(options: G14SecretOptions): string {
function v02MasterAdminApiKeySecretScript(options: G14SecretOptions, spec: HwlabRuntimeLaneSpec): string {
const namespace = spec.runtimeNamespace;
const name = runtimeLaneMasterAdminApiKeySecretName(spec);
const fieldManager = runtimeLaneSecretFieldManager(spec);
return [
"set +e",
`namespace=${shellQuote(V02_RUNTIME_NAMESPACE)}`,
`name=${shellQuote(V02_MASTER_ADMIN_API_KEY_SECRET)}`,
`namespace=${shellQuote(namespace)}`,
`name=${shellQuote(name)}`,
`api_key_name=${shellQuote(V02_MASTER_ADMIN_API_KEY_SECRET_KEY)}`,
`action_request=${shellQuote(options.action)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
`field_manager=${shellQuote(V02_SECRET_FIELD_MANAGER)}`,
`field_manager=${shellQuote(fieldManager)}`,
"preset=master-server-admin-api-key",
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }",
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }",
@@ -4504,7 +4734,7 @@ function v02SecretStatusFromText(text: string, commandOk: boolean, exitCode: num
typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 &&
typeof afterUriBytes === "number" && afterUriBytes > 0 &&
typeof afterPasswordBytes === "number" && afterPasswordBytes > 0;
const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t";
const databaseHealthy = fields.dbMode === "primary" || (fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t");
const healthy = keysHealthy && databaseHealthy;
return {
ok: commandOk && healthy,
@@ -4515,6 +4745,7 @@ function v02SecretStatusFromText(text: string, commandOk: boolean, exitCode: num
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
dbMode: fields.dbMode || "dedicated",
before: {
exists: fields.beforeExists === "yes",
authnPresharedKey: { keyPresent: fields.beforeAuthnPresent === "yes", valueBytes: numericField(fields.beforeAuthnBytes) },
@@ -4538,6 +4769,7 @@ function v02SecretStatusFromText(text: string, commandOk: boolean, exitCode: num
},
},
postgresAdminSecretPresent: fields.postgresAdminSecretPresent === "yes",
postgresSecret: fields.postgresSecret || null,
dbName: fields.dbName || V02_OPENFGA_DB_NAME,
dbUser: fields.dbUser || V02_OPENFGA_DB_USER,
applyExitCode: numericField(fields.applyExitCode),
@@ -4566,9 +4798,10 @@ function v02SecretStatusFromText(text: string, commandOk: boolean, exitCode: num
}
function runG14Secret(options: G14SecretOptions): Record<string, unknown> {
const spec = hwlabRuntimeLaneSpec(options.lane);
const script = v02SecretScript(options);
const localAdminApiKey = options.preset === "master-server-admin-api-key"
? readOrCreateLocalMasterAdminApiKey(options.action !== "ensure" || options.dryRun)
? readOrCreateLocalMasterAdminApiKey(spec, options.action !== "ensure" || options.dryRun)
: null;
const result = options.preset === "master-server-admin-api-key"
? g14K3sInlineScriptWithInput(script, localAdminApiKey?.key ?? "", options.timeoutSeconds * 1000 + 2000)
@@ -4579,9 +4812,9 @@ function runG14Secret(options: G14SecretOptions): Record<string, unknown> {
const ok = dryRunOk || deleteDryRunOk ? true : status.ok === true;
return {
ok,
command: `hwlab g14 secret ${options.action} --lane v02`,
command: `hwlab g14 secret ${options.action} --lane ${options.lane}`,
lane: options.lane,
namespace: V02_RUNTIME_NAMESPACE,
namespace: spec.runtimeNamespace,
secret: options.name,
key: options.key ?? null,
preset: options.preset,
@@ -4597,7 +4830,7 @@ function runG14Secret(options: G14SecretOptions): Record<string, unknown> {
? undefined
: options.action === "delete"
? undefined
: { ensure: `bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` },
: { ensure: `bun scripts/cli.ts hwlab g14 secret ensure --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` },
};
}
@@ -8433,14 +8666,18 @@ export function hwlabG14Help(): Record<string, unknown> {
"bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --source-commit <full-sha>",
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --dry-run",
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm",
"bun scripts/cli.ts hwlab g14 control-plane refresh --lane v02 --dry-run",
"bun scripts/cli.ts hwlab g14 control-plane refresh --lane v02 --confirm",
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm",
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm --wait",
"bun scripts/cli.ts hwlab g14 control-plane status --lane v03",
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v03 --dry-run",
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v03 --confirm",
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v03 --dry-run",
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v03 --confirm",
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v03 --confirm --wait",
"bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03",
"bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm --wait",
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --dry-run",
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --confirm",
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --pipeline-run hwlab-v02-ci-poll-<short-sha> --dry-run",
@@ -8453,13 +8690,18 @@ export function hwlabG14Help(): Record<string, unknown> {
"bun scripts/cli.ts hwlab g14 secret status --lane v02 --name hwlab-v02-openfga",
"bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-openfga --dry-run",
"bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-openfga --confirm",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-openfga --dry-run",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-openfga --confirm",
"bun scripts/cli.ts hwlab g14 secret status --lane v02 --name hwlab-v02-master-server-admin-api-key",
"bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-master-server-admin-api-key --confirm",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm",
"bun scripts/cli.ts hwlab g14 secret delete --lane v02 --name <obsolete-hwlab-v02-secret> --dry-run",
"bun scripts/cli.ts hwlab g14 secret delete --lane v02 --name <obsolete-hwlab-v02-secret> --confirm",
"bun scripts/cli.ts hwlab g14 git-mirror status",
"bun scripts/cli.ts hwlab g14 git-mirror apply --lane v02 --confirm",
"bun scripts/cli.ts hwlab g14 git-mirror apply --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes git-mirror apply --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab g14 git-mirror sync --confirm",
"bun scripts/cli.ts hwlab g14 git-mirror flush --confirm",
"bun scripts/cli.ts hwlab g14 git-mirror sync --confirm --wait",
@@ -8480,7 +8722,7 @@ export function hwlabG14Help(): Record<string, unknown> {
"bun scripts/cli.ts hwlab g14 upstream-image ensure --name openfga --tag v1.17.0 --confirm",
"bun scripts/cli.ts job status <jobId> --tail-bytes 30000",
],
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, runtime lane v02/v03 control-plane apply/status/trigger entry, v0.2 runtime SecretRef bootstrap, devops-infra git mirror and observability maintenance, controlled CI tools image build/status entry, and allowlisted upstream image mirroring. The public monitor starts a fire-and-forget job. Default monitor lane is base=G14; --lane v02 monitors base=v0.2 PRs, waits for GitHub preflight/CI readiness, automatically merges ready PRs without waiting for other active v0.2 PipelineRuns, triggers v0.2 CD with latest-only GitOps writeback, flushes the git mirror when needed, and posts deduplicated PR comments for pending, blocked/conflict, success, superseded, failure, or timeout states. confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane v02 keeps the full closeout/cleanup/runtime-migration verdict path; v03+ uses the runtime lane spec for Tekton/Argo apply, status visibility, and commit-pinned PipelineRun trigger. secret status/ensure is the standard v0.2 runtime SecretRef bootstrap path; it never reads or prints secret values. upstream-image status/ensure only mirrors allowlisted upstream runtime images into the G14 local registry. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob. observability status/apply/query/targets/boundary/closeout owns the shared Prometheus Operator and Prometheus instance in devops-infra, adds bounded PromQL assertions and semantic closeout summaries, while HWLAB lane manifests own only ServiceMonitor and PrometheusRule objects.",
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, node-scoped runtime lane v03 control-plane apply/status/refresh/trigger entry, runtime lane SecretRef bootstrap, devops-infra git mirror and observability maintenance, controlled CI tools image build/status entry, and allowlisted upstream image mirroring. The public monitor starts a fire-and-forget job. Default monitor lane is base=G14; --lane v02 monitors base=v0.2 PRs, waits for GitHub preflight/CI readiness, automatically merges ready PRs without waiting for other active v0.2 PipelineRuns, triggers v0.2 CD with latest-only GitOps writeback, flushes the git mirror when needed, and posts deduplicated PR comments for pending, blocked/conflict, success, superseded, failure, or timeout states. confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane v02 keeps the full closeout/cleanup/runtime-migration verdict path; v03+ is advertised through `hwlab nodes ... --node <node-id> --lane vNN` so node identity remains configuration data instead of a command family. secret status/ensure is the standard runtime lane SecretRef bootstrap path for OpenFGA and master admin API key; it never reads or prints secret values. upstream-image status/ensure only mirrors allowlisted upstream runtime images into the G14 local registry. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob. observability status/apply/query/targets/boundary/closeout owns the shared Prometheus Operator and Prometheus instance in devops-infra, adds bounded PromQL assertions and semantic closeout summaries, while HWLAB lane manifests own only ServiceMonitor and PrometheusRule objects.",
defaults: {
repo: HWLAB_REPO,
base: G14_SOURCE_BRANCH,
@@ -92,7 +92,7 @@ export interface HwlabRuntimeLaneSpec {
readonly downloadProfile: HwlabDownloadProfileSpec;
}
export const HWLAB_G14_LANE_CONFIG_PATH = "config/hwlab-g14-lanes.yaml";
export const HWLAB_NODE_LANE_CONFIG_PATH = "config/hwlab-node-lanes.yaml";
interface HwlabLaneConfig {
readonly id: HwlabRuntimeLane;
@@ -120,7 +120,7 @@ interface HwlabLaneConfig {
readonly public: { readonly webUrl: string; readonly apiUrl: string };
}
interface HwlabG14LaneConfig {
interface HwlabNodeLaneConfig {
readonly requiredNoProxy: readonly string[];
readonly nodes: Record<string, HwlabRuntimeNodeSpec>;
readonly lanes: Record<HwlabRuntimeLane, HwlabLaneConfig>;
@@ -283,13 +283,13 @@ function laneConfig(id: HwlabRuntimeLane, raw: Record<string, unknown>): HwlabLa
};
}
function readHwlabG14LaneConfig(): HwlabG14LaneConfig {
const path = rootPath(HWLAB_G14_LANE_CONFIG_PATH);
function readHwlabNodeLaneConfig(): HwlabNodeLaneConfig {
const path = rootPath(HWLAB_NODE_LANE_CONFIG_PATH);
const raw = readFileSync(path, "utf8");
const parsed = asRecord(Bun.YAML.parse(raw) as unknown, HWLAB_G14_LANE_CONFIG_PATH);
const requiredNoProxy = stringArrayField(parsed, "requiredNoProxy", HWLAB_G14_LANE_CONFIG_PATH);
const parsed = asRecord(Bun.YAML.parse(raw) as unknown, HWLAB_NODE_LANE_CONFIG_PATH);
const requiredNoProxy = stringArrayField(parsed, "requiredNoProxy", HWLAB_NODE_LANE_CONFIG_PATH);
for (const required of ["hyueapi.com", ".hyueapi.com"]) {
if (!requiredNoProxy.includes(required)) throw new Error(`${HWLAB_G14_LANE_CONFIG_PATH}.requiredNoProxy must include ${required}`);
if (!requiredNoProxy.includes(required)) throw new Error(`${HWLAB_NODE_LANE_CONFIG_PATH}.requiredNoProxy must include ${required}`);
}
const nodes = Object.fromEntries(sortedRecordEntries(parsed.nodes, "nodes").map(([id, item]) => [id, nodeConfig(id, item)]));
const networkProfiles = Object.fromEntries(
@@ -312,12 +312,12 @@ function readHwlabG14LaneConfig(): HwlabG14LaneConfig {
return { requiredNoProxy, nodes, lanes, networkProfiles, downloadProfiles };
}
const HWLAB_G14_LANE_CONFIG = readHwlabG14LaneConfig();
const HWLAB_NODE_LANE_CONFIG = readHwlabNodeLaneConfig();
function buildRuntimeLaneSpec(config: HwlabLaneConfig): HwlabRuntimeLaneSpec {
const node = HWLAB_G14_LANE_CONFIG.nodes[config.node];
const networkProfile = HWLAB_G14_LANE_CONFIG.networkProfiles[node.networkProfileId];
const downloadProfile = HWLAB_G14_LANE_CONFIG.downloadProfiles[node.downloadProfileId];
const node = HWLAB_NODE_LANE_CONFIG.nodes[config.node];
const networkProfile = HWLAB_NODE_LANE_CONFIG.networkProfiles[node.networkProfileId];
const downloadProfile = HWLAB_NODE_LANE_CONFIG.downloadProfiles[node.downloadProfileId];
return {
lane: config.id,
nodeId: node.id,
@@ -358,7 +358,7 @@ function buildRuntimeLaneSpec(config: HwlabLaneConfig): HwlabRuntimeLaneSpec {
}
const RUNTIME_LANE_SPECS = Object.fromEntries(
Object.values(HWLAB_G14_LANE_CONFIG.lanes).map((config) => [config.id, buildRuntimeLaneSpec(config)]),
Object.values(HWLAB_NODE_LANE_CONFIG.lanes).map((config) => [config.id, buildRuntimeLaneSpec(config)]),
) as Record<HwlabRuntimeLane, HwlabRuntimeLaneSpec>;
export function isHwlabRuntimeLane(value: string): value is HwlabRuntimeLane {
@@ -374,13 +374,13 @@ export function hwlabRuntimeLaneIds(): HwlabRuntimeLane[] {
}
export function hwlabRuntimeNodeIds(): string[] {
return Object.keys(HWLAB_G14_LANE_CONFIG.nodes);
return Object.keys(HWLAB_NODE_LANE_CONFIG.nodes);
}
export function hwlabRuntimeLaneConfigPath(): string {
return HWLAB_G14_LANE_CONFIG_PATH;
return HWLAB_NODE_LANE_CONFIG_PATH;
}
export function hwlabRequiredNoProxyEntries(): string[] {
return [...HWLAB_G14_LANE_CONFIG.requiredNoProxy];
return [...HWLAB_NODE_LANE_CONFIG.requiredNoProxy];
}
+149 -3
View File
@@ -1,9 +1,13 @@
import { existsSync, readFileSync } from "node:fs";
import { repoRoot, type Config } from "./config";
import { runCommand, type CommandResult } from "./command";
import { startJob } from "./jobs";
import { runHwlabG14Command } from "./hwlab-g14";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane } from "./hwlab-node-lanes";
type SecretAction = "status" | "ensure";
type SecretPreset = "openfga" | "master-server-admin-api-key";
type DelegatedNodeDomain = "control-plane" | "git-mirror";
interface NodeSecretOptions {
action: SecretAction;
@@ -41,8 +45,11 @@ const OPENFGA_POSTGRES_PASSWORD_KEY = "postgres-password";
export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise<Record<string, unknown>> {
if (args.length === 0 || args.includes("--help") || args.includes("-h")) return hwlabNodeHelp();
const [domain] = args;
if (domain === "control-plane" || domain === "git-mirror") {
return runNodeDelegatedDomain(_config, domain, args.slice(1));
}
if (domain !== "secret") {
return { ok: false, command: `hwlab node ${domain ?? ""}`.trim(), message: "supported commands: hwlab node secret status|ensure" };
return { ok: false, command: `hwlab nodes ${domain ?? ""}`.trim(), message: "supported commands: hwlab nodes control-plane, hwlab nodes git-mirror, hwlab nodes secret" };
}
const options = parseSecretOptions(args.slice(1));
return runNodeSecret(options);
@@ -53,7 +60,13 @@ export function hwlabNodeHelp(): Record<string, unknown> {
ok: true,
command: "hwlab nodes",
description: "Node/lane oriented HWLAB operations. G14 is a node id value passed by --node, not a command family.",
configPath: hwlabRuntimeLaneConfigPath(),
examples: [
"bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03",
"bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes git-mirror status --node G14 --lane v03",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-openfga --confirm",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm",
@@ -61,14 +74,123 @@ export function hwlabNodeHelp(): Record<string, unknown> {
};
}
async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise<Record<string, unknown>> {
const scoped = parseNodeScopedDelegatedOptions(domain, args);
if (domain === "control-plane" && scoped.action === "trigger-current" && scoped.confirm && !scoped.dryRun && !scoped.wait) {
return startNodeDelegatedJob(scoped);
}
if (domain === "git-mirror" && (scoped.action === "sync" || scoped.action === "flush") && scoped.confirm && !scoped.dryRun && !scoped.wait) {
return startNodeDelegatedJob(scoped);
}
const delegatedArgs = stripOption(args, "--node");
const result = await runHwlabG14Command(config, [domain, ...delegatedArgs]);
return rewriteDelegatedNodeResult(result, scoped);
}
function parseNodeScopedDelegatedOptions(domain: DelegatedNodeDomain, args: string[]): {
domain: DelegatedNodeDomain;
action: string;
node: string;
lane: HwlabRuntimeLane;
confirm: boolean;
dryRun: boolean;
wait: boolean;
timeoutSeconds: number;
originalArgs: string[];
} {
const [actionRaw] = args;
if (typeof actionRaw !== "string" || actionRaw.startsWith("--")) throw new Error(`${domain} usage: ${domain} ACTION --node NODE --lane vNN [--dry-run|--confirm]`);
const node = requiredOption(args, "--node");
assertNodeId(node);
const laneRaw = requiredOption(args, "--lane");
if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`--lane must be one of v02, v03; got ${laneRaw}`);
const spec = hwlabRuntimeLaneSpec(laneRaw);
if (spec.nodeId !== node) throw new Error(`lane ${laneRaw} is configured for node ${spec.nodeId}; got --node ${node}`);
const confirm = args.includes("--confirm");
const dryRun = args.includes("--dry-run");
if (confirm && dryRun) throw new Error(`${domain} accepts only one of --confirm or --dry-run`);
return {
domain,
action: actionRaw,
node,
lane: laneRaw,
confirm,
dryRun,
wait: args.includes("--wait"),
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 1800, 3600),
originalArgs: [...args],
};
}
function startNodeDelegatedJob(options: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const commandArgs = [
"hwlab",
"nodes",
options.domain,
...stripOptions(options.originalArgs, ["--node", "--lane", "--confirm", "--dry-run", "--wait", "--timeout-seconds"]),
"--node",
options.node,
"--lane",
options.lane,
"--confirm",
"--timeout-seconds",
String(options.timeoutSeconds),
"--wait",
];
const command = ["bun", "scripts/cli.ts", ...commandArgs];
const job = startJob(
`hwlab_nodes_${options.lane}_${options.domain}_${options.action}`,
command,
`Run HWLAB ${options.lane} ${options.domain} ${options.action} for node ${options.node}`,
);
return {
ok: true,
command: `hwlab nodes ${options.domain} ${options.action} --node ${options.node} --lane ${options.lane}`,
node: options.node,
lane: options.lane,
mode: "async-job",
reason: "confirmed control-plane/mirror actions can spend tens of seconds on remote work; default is fire-and-forget to avoid silent blocking",
job,
statusCommand: `bun scripts/cli.ts job status ${job.id}`,
waitCommand: command.join(" "),
};
}
function rewriteDelegatedNodeResult(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const rewritten = rewriteDelegatedNodeValue(value, scoped);
const result = typeof rewritten === "object" && rewritten !== null && !Array.isArray(rewritten) ? rewritten as Record<string, unknown> : { value: rewritten };
return {
...result,
node: scoped.node,
commandFamily: "hwlab nodes",
};
}
function rewriteDelegatedNodeValue(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): unknown {
if (typeof value === "string") return rewriteDelegatedNodeString(value, scoped);
if (Array.isArray(value)) return value.map((item) => rewriteDelegatedNodeValue(item, scoped));
if (typeof value !== "object" || value === null) return value;
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewriteDelegatedNodeValue(item, scoped)]));
}
function rewriteDelegatedNodeString(value: string, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
const replaceCommand = (text: string, domain: DelegatedNodeDomain) => {
const escapedDomain = domain.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
return text
.replace(new RegExp(`bun scripts/cli\\.ts hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `bun scripts/cli.ts hwlab nodes ${domain} $1 --node ${scoped.node}`)
.replace(new RegExp(`hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `hwlab nodes ${domain} $1 --node ${scoped.node}`);
};
return replaceCommand(replaceCommand(value, "control-plane"), "git-mirror");
}
function parseSecretOptions(args: string[]): NodeSecretOptions {
const [actionRaw] = args;
if (actionRaw !== "status" && actionRaw !== "ensure") {
throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key [--dry-run|--confirm]");
}
const node = optionValue(args, "--node") ?? "G14";
const node = requiredOption(args, "--node");
assertNodeId(node);
const lane = optionValue(args, "--lane") ?? "v03";
const lane = requiredOption(args, "--lane");
assertLane(lane);
const spec = runtimeSecretSpec({ node, lane });
const name = optionValue(args, "--name") ?? spec.openFgaSecret;
@@ -448,6 +570,30 @@ function optionValue(args: string[], name: string): string | undefined {
return value;
}
function requiredOption(args: string[], name: string): string {
const value = optionValue(args, name);
if (value === undefined) throw new Error(`${name} is required`);
return value;
}
function stripOption(args: string[], name: string): string[] {
return stripOptions(args, [name]);
}
function stripOptions(args: string[], names: readonly string[]): string[] {
const remove = new Set(names);
const without: string[] = [];
for (let index = 0; index < args.length; index += 1) {
const arg = args[index] ?? "";
if (remove.has(arg)) {
if (arg !== "--confirm" && arg !== "--dry-run" && arg !== "--wait") index += 1;
continue;
}
without.push(arg);
}
return without;
}
function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
const raw = optionValue(args, name);
if (raw === undefined) return defaultValue;