feat: extend G14 runtime automation
This commit is contained in:
Codex
+141
-8
@@ -5,8 +5,8 @@ import { startJob } from "./jobs";
|
||||
import { runHwlabG14Command } from "./hwlab-g14";
|
||||
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane } from "./hwlab-node-lanes";
|
||||
|
||||
type SecretAction = "status" | "ensure";
|
||||
type SecretPreset = "openfga" | "master-server-admin-api-key" | "code-agent-provider" | "cloud-api-db";
|
||||
type SecretAction = "status" | "ensure" | "cleanup-owned-postgres";
|
||||
type SecretPreset = "openfga" | "master-server-admin-api-key" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup";
|
||||
type DelegatedNodeDomain = "control-plane" | "git-mirror";
|
||||
|
||||
interface NodeSecretOptions {
|
||||
@@ -86,6 +86,8 @@ export function hwlabNodeHelp(): Record<string, unknown> {
|
||||
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db",
|
||||
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-cloud-api-v03-db --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider",
|
||||
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm",
|
||||
],
|
||||
@@ -203,8 +205,8 @@ function rewriteDelegatedNodeString(value: string, scoped: ReturnType<typeof par
|
||||
|
||||
function parseSecretOptions(args: string[]): NodeSecretOptions {
|
||||
const [actionRaw] = args;
|
||||
if (actionRaw !== "status" && actionRaw !== "ensure") {
|
||||
throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key|hwlab-cloud-api-vNN-db|hwlab-vNN-code-agent-provider [--dry-run|--confirm]");
|
||||
if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "cleanup-owned-postgres") {
|
||||
throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key|hwlab-cloud-api-vNN-db|hwlab-vNN-code-agent-provider [--dry-run|--confirm] | cleanup-owned-postgres --node NODE --lane vNN [--dry-run|--confirm]");
|
||||
}
|
||||
const node = requiredOption(args, "--node");
|
||||
assertNodeId(node);
|
||||
@@ -216,6 +218,21 @@ function parseSecretOptions(args: string[]): NodeSecretOptions {
|
||||
const confirm = args.includes("--confirm");
|
||||
const explicitDryRun = args.includes("--dry-run");
|
||||
if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run");
|
||||
if (actionRaw === "cleanup-owned-postgres") {
|
||||
if (lane === "v02") throw new Error("secret cleanup-owned-postgres is only for v0.3+ lanes after migration to G14 platform Postgres");
|
||||
if (key !== undefined) throw new Error("secret cleanup-owned-postgres does not accept --key");
|
||||
if (name !== spec.openFgaSecret && name !== spec.postgresSecret) throw new Error(`secret cleanup-owned-postgres for --lane ${lane} targets ${spec.postgresSecret}; omit --name or pass --name ${spec.postgresSecret}`);
|
||||
return {
|
||||
action: actionRaw,
|
||||
node,
|
||||
lane,
|
||||
name: spec.postgresSecret,
|
||||
preset: "owned-postgres-cleanup",
|
||||
confirm,
|
||||
dryRun: explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
||||
};
|
||||
}
|
||||
if (name === spec.masterAdminApiKeySecret) {
|
||||
if (key !== undefined && key !== MASTER_ADMIN_API_KEY_KEY) throw new Error(`secret ${name} supports only key ${MASTER_ADMIN_API_KEY_KEY}`);
|
||||
return {
|
||||
@@ -317,11 +334,14 @@ function runNodeSecret(options: NodeSecretOptions): Record<string, unknown> {
|
||||
? masterAdminApiKeySecretScript(options, spec)
|
||||
: options.preset === "cloud-api-db"
|
||||
? cloudApiDbSecretScript(options, spec)
|
||||
: codeAgentProviderSecretScript(options, spec);
|
||||
: options.preset === "owned-postgres-cleanup"
|
||||
? ownedPostgresCleanupScript(options, spec)
|
||||
: codeAgentProviderSecretScript(options, spec);
|
||||
const result = runTransScript(options.node, script, input, options.timeoutSeconds);
|
||||
const status = secretStatusFromText(statusText(result), result.exitCode === 0, result.exitCode, result.stderr, spec);
|
||||
const dryRunOk = options.action === "ensure" && options.dryRun && result.exitCode === 0;
|
||||
const ok = dryRunOk ? true : status.ok === true;
|
||||
const cleanupDryRunOk = options.action === "cleanup-owned-postgres" && options.dryRun && result.exitCode === 0;
|
||||
const ok = dryRunOk || cleanupDryRunOk ? true : status.ok === true;
|
||||
return {
|
||||
ok,
|
||||
command: `hwlab nodes secret ${options.action}`,
|
||||
@@ -331,13 +351,15 @@ function runNodeSecret(options: NodeSecretOptions): Record<string, unknown> {
|
||||
secret: options.name,
|
||||
key: options.key ?? null,
|
||||
preset: options.preset,
|
||||
mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : "confirmed-ensure",
|
||||
mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "cleanup-owned-postgres" ? "confirmed-delete" : "confirmed-ensure",
|
||||
status,
|
||||
mutation: status.mutation === true,
|
||||
result: compactCommandResult(result),
|
||||
valuesRedacted: true,
|
||||
next: ok && options.action === "status" ? undefined : {
|
||||
ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm`,
|
||||
ensure: options.action === "cleanup-owned-postgres"
|
||||
? `bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node ${options.node} --lane ${options.lane} --confirm`
|
||||
: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm`,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -346,6 +368,76 @@ function runTransScript(node: string, script: string, input: string, timeoutSeco
|
||||
return runCommand(["/root/.local/bin/trans", `${node}:k3s`, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
|
||||
}
|
||||
|
||||
function ownedPostgresCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
||||
const pvc = `data-${spec.postgresSecret}-0`;
|
||||
const platformService = "g14-platform-postgres";
|
||||
return [
|
||||
"set +e",
|
||||
`namespace=${shellQuote(spec.namespace)}`,
|
||||
`postgres_secret=${shellQuote(spec.postgresSecret)}`,
|
||||
`postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`,
|
||||
`pvc=${shellQuote(pvc)}`,
|
||||
`platform_service=${shellQuote(platformService)}`,
|
||||
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
||||
"preset=owned-postgres-cleanup",
|
||||
"exists_flag() { kind=\"$1\"; item=\"$2\"; kubectl -n \"$namespace\" get \"$kind\" \"$item\" >/dev/null 2>&1 && printf yes || printf no; }",
|
||||
"pv_name() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.spec.volumeName}' 2>/dev/null; }",
|
||||
"phase_of_pvc() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.status.phase}' 2>/dev/null; }",
|
||||
"before_secret_exists=$(exists_flag secret \"$postgres_secret\")",
|
||||
"before_pvc_exists=$(exists_flag pvc \"$pvc\")",
|
||||
"before_pvc_phase=$(phase_of_pvc)",
|
||||
"before_pv=$(pv_name)",
|
||||
"before_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")",
|
||||
"platform_service_exists=$(exists_flag service \"$platform_service\")",
|
||||
"action=observed",
|
||||
"mutation=false",
|
||||
"delete_secret_exit=",
|
||||
"delete_pvc_exit=",
|
||||
"if [ \"$dry_run\" = true ]; then",
|
||||
" if [ \"$before_secret_exists\" = yes ] || [ \"$before_pvc_exists\" = yes ]; then action=would-delete; else action=already-absent; fi",
|
||||
"else",
|
||||
" kubectl -n \"$namespace\" delete secret \"$postgres_secret\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-secret-delete.out 2>/tmp/hwlab-owned-postgres-secret-delete.err",
|
||||
" delete_secret_exit=$?",
|
||||
" kubectl -n \"$namespace\" delete pvc \"$pvc\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-pvc-delete.out 2>/tmp/hwlab-owned-postgres-pvc-delete.err",
|
||||
" delete_pvc_exit=$?",
|
||||
" if [ \"$delete_secret_exit\" -eq 0 ] && [ \"$delete_pvc_exit\" -eq 0 ]; then",
|
||||
" if [ \"$before_secret_exists\" = yes ] || [ \"$before_pvc_exists\" = yes ]; then action=deleted; mutation=true; else action=already-absent; fi",
|
||||
" else",
|
||||
" action=delete-failed",
|
||||
" fi",
|
||||
"fi",
|
||||
"after_secret_exists=$(exists_flag secret \"$postgres_secret\")",
|
||||
"after_pvc_exists=$(exists_flag pvc \"$pvc\")",
|
||||
"after_pvc_phase=$(phase_of_pvc)",
|
||||
"after_pv=$(pv_name)",
|
||||
"after_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")",
|
||||
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
||||
"printf 'secret\\t%s\\n' \"$postgres_secret\"",
|
||||
"printf 'pvc\\t%s\\n' \"$pvc\"",
|
||||
"printf 'preset\\t%s\\n' \"$preset\"",
|
||||
"printf 'action\\t%s\\n' \"$action\"",
|
||||
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
||||
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
||||
"printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
|
||||
"printf 'beforePvcExists\\t%s\\n' \"$before_pvc_exists\"",
|
||||
"printf 'beforePvcPhase\\t%s\\n' \"$before_pvc_phase\"",
|
||||
"printf 'beforePersistentVolume\\t%s\\n' \"$before_pv\"",
|
||||
"printf 'beforeStatefulSetExists\\t%s\\n' \"$before_statefulset_exists\"",
|
||||
"printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"",
|
||||
"printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
|
||||
"printf 'afterPvcExists\\t%s\\n' \"$after_pvc_exists\"",
|
||||
"printf 'afterPvcPhase\\t%s\\n' \"$after_pvc_phase\"",
|
||||
"printf 'afterPersistentVolume\\t%s\\n' \"$after_pv\"",
|
||||
"printf 'afterStatefulSetExists\\t%s\\n' \"$after_statefulset_exists\"",
|
||||
"printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"",
|
||||
"printf 'deletePvcExitCode\\t%s\\n' \"$delete_pvc_exit\"",
|
||||
"if [ \"$platform_service_exists\" != yes ]; then exit 44; fi",
|
||||
"if [ \"$before_statefulset_exists\" = yes ] || [ \"$after_statefulset_exists\" = yes ]; then exit 45; fi",
|
||||
"if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi",
|
||||
"if [ -n \"$delete_pvc_exit\" ] && [ \"$delete_pvc_exit\" != 0 ]; then exit \"$delete_pvc_exit\"; fi",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
||||
return [
|
||||
"set +e",
|
||||
@@ -803,6 +895,47 @@ function cloudApiDbSecretScript(options: NodeSecretOptions, spec: RuntimeSecretS
|
||||
|
||||
function secretStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record<string, unknown> {
|
||||
const fields = keyValueLinesFromText(text);
|
||||
if (fields.preset === "owned-postgres-cleanup") {
|
||||
const absent = fields.afterSecretExists !== "yes" && fields.afterPvcExists !== "yes";
|
||||
const oldWorkloadAbsent = fields.afterStatefulSetExists !== "yes";
|
||||
const platformServiceReady = fields.platformServiceExists === "yes";
|
||||
return {
|
||||
ok: commandOk && absent && oldWorkloadAbsent && platformServiceReady,
|
||||
namespace: fields.namespace || spec.namespace,
|
||||
secret: fields.secret || spec.postgresSecret,
|
||||
pvc: fields.pvc || `data-${spec.postgresSecret}-0`,
|
||||
preset: "owned-postgres-cleanup",
|
||||
action: fields.action || null,
|
||||
dryRun: fields.dryRun === "true",
|
||||
mutation: fields.mutation === "true",
|
||||
before: {
|
||||
secretExists: fields.beforeSecretExists === "yes",
|
||||
pvcExists: fields.beforePvcExists === "yes",
|
||||
pvcPhase: fields.beforePvcPhase || null,
|
||||
persistentVolume: fields.beforePersistentVolume || null,
|
||||
statefulSetExists: fields.beforeStatefulSetExists === "yes",
|
||||
},
|
||||
after: {
|
||||
secretExists: fields.afterSecretExists === "yes",
|
||||
pvcExists: fields.afterPvcExists === "yes",
|
||||
pvcPhase: fields.afterPvcPhase || null,
|
||||
persistentVolume: fields.afterPersistentVolume || null,
|
||||
statefulSetExists: fields.afterStatefulSetExists === "yes",
|
||||
},
|
||||
platformService: {
|
||||
name: "g14-platform-postgres",
|
||||
exists: platformServiceReady,
|
||||
},
|
||||
deleteSecretExitCode: numericField(fields.deleteSecretExitCode),
|
||||
deletePvcExitCode: numericField(fields.deletePvcExitCode),
|
||||
exitCode,
|
||||
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
||||
valuesRedacted: true,
|
||||
summary: absent
|
||||
? `${fields.secret || spec.postgresSecret} and ${fields.pvc || `data-${spec.postgresSecret}-0`} absent`
|
||||
: `${fields.secret || spec.postgresSecret} or ${fields.pvc || `data-${spec.postgresSecret}-0`} still exists`,
|
||||
};
|
||||
}
|
||||
if (fields.preset === "master-server-admin-api-key") {
|
||||
const afterBytes = numericField(fields.afterApiKeyBytes);
|
||||
const healthy = fields.afterExists === "yes" && fields.afterApiKeyPresent === "yes" && typeof afterBytes === "number" && afterBytes > 0;
|
||||
|
||||
Reference in New Issue
Block a user