refactor: split control-plane cli modules
This commit is contained in:
Codex
@@ -0,0 +1,927 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. public-exposure module for scripts/src/hwlab-node-impl.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-node-impl.ts:11201-12084 for #903.
|
||||
|
||||
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
||||
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
||||
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand, type CommandResult } from "../command";
|
||||
import { startJob } from "../jobs";
|
||||
import { classifySshTcpPoolFailure } from "../ssh";
|
||||
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
||||
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
||||
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
||||
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
||||
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
||||
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
|
||||
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
||||
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
||||
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
|
||||
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
||||
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
||||
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
||||
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
||||
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
||||
import type { RenderedCliResult } from "../output";
|
||||
|
||||
import type { NodePublicExposureOptions, NodeSecretOptions, RuntimeSecretSpec } from "./entry";
|
||||
import { BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY, BOOTSTRAP_ADMIN_SOURCE_NAMESPACE, BOOTSTRAP_ADMIN_SOURCE_SECRET, CLOUD_API_DB_KEY, CODE_AGENT_PROVIDER_OPENAI_KEY, CODE_AGENT_PROVIDER_OPENCODE_KEY, CODE_AGENT_PROVIDER_SOURCE_NAMESPACE, CODE_AGENT_PROVIDER_SOURCE_SECRET, MASTER_ADMIN_API_KEY_KEY, OPENFGA_AUTHN_KEY, OPENFGA_DATASTORE_URI_KEY, OPENFGA_POSTGRES_PASSWORD_KEY } from "./entry";
|
||||
import { transPath } from "./runtime-common";
|
||||
import { bootstrapAdminSecretScript, cloudApiDbSecretScript, codeAgentProviderSecretScript, masterAdminApiKeySecretScript, obsoletePlatformDbCleanupScript, obsoletePlatformDbStatusFromText, obsoleteSecretCleanupScript, openFgaSecretScript, ownedPostgresCleanupScript, platformDbSecretStatusScript, secretStatusFromText } from "./secret-scripts";
|
||||
import { assertLane, assertNodeId, compactCommandResult, keyValueLinesFromText, numericField, optionValue, positiveIntegerOption, readMasterAdminApiKey, requiredOption, shellQuote, statusText } from "./utils";
|
||||
import { parseEnvFile, readBootstrapAdminSecretMaterial, syncNodeExternalPostgresSecrets } from "./web-probe";
|
||||
|
||||
export function isSafeWebProbeScriptRunDir(value: string | null): value is string {
|
||||
return typeof value === "string"
|
||||
&& value.length > 0
|
||||
&& !value.includes("\0")
|
||||
&& !value.includes("..")
|
||||
&& /\.state\/web-probe-script\/run\.[A-Za-z0-9]+$/u.test(value);
|
||||
}
|
||||
|
||||
export function isSafeWebProbeScriptReportPath(value: string | null): value is string {
|
||||
return typeof value === "string"
|
||||
&& isSafeWebProbeScriptArtifactPath(value)
|
||||
&& value.endsWith("/web-probe-script-report.json");
|
||||
}
|
||||
|
||||
export function isSafeWebProbeScriptArtifactPath(value: string): boolean {
|
||||
return typeof value === "string"
|
||||
&& value.length > 0
|
||||
&& !value.includes("\0")
|
||||
&& !value.includes("..")
|
||||
&& /\.state\/web-probe-script\/run\.[A-Za-z0-9]+\/[^/]+[.](?:png|json)$/u.test(value);
|
||||
}
|
||||
|
||||
export function parseSecretOptions(args: string[]): NodeSecretOptions {
|
||||
const [actionRaw] = args;
|
||||
if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "cleanup-owned-postgres" && actionRaw !== "cleanup-obsolete") {
|
||||
throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key|hwlab-vNN-bootstrap-admin|hwlab-cloud-api-vNN-db|hwlab-vNN-code-agent-provider [--dry-run|--confirm] | cleanup-owned-postgres --node NODE --lane vNN [--dry-run|--confirm] | cleanup-obsolete --node NODE --lane vNN --name hwpod-vNN-db [--dry-run|--confirm]");
|
||||
}
|
||||
const node = requiredOption(args, "--node");
|
||||
assertNodeId(node);
|
||||
const lane = requiredOption(args, "--lane");
|
||||
assertLane(lane);
|
||||
const spec = runtimeSecretSpec({ node, lane });
|
||||
const name = optionValue(args, "--name") ?? spec.openFgaSecret;
|
||||
const key = optionValue(args, "--key");
|
||||
const confirm = args.includes("--confirm");
|
||||
const explicitDryRun = args.includes("--dry-run");
|
||||
if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run");
|
||||
if (actionRaw === "cleanup-owned-postgres") {
|
||||
if (lane === "v02") throw new Error("secret cleanup-owned-postgres is only for v0.3+ lanes after migration to G14 platform Postgres");
|
||||
if (key !== undefined) throw new Error("secret cleanup-owned-postgres does not accept --key");
|
||||
if (name !== spec.openFgaSecret && name !== spec.postgresSecret) throw new Error(`secret cleanup-owned-postgres for --lane ${lane} targets ${spec.postgresSecret}; omit --name or pass --name ${spec.postgresSecret}`);
|
||||
return {
|
||||
action: actionRaw,
|
||||
node,
|
||||
lane,
|
||||
name: spec.postgresSecret,
|
||||
preset: "owned-postgres-cleanup",
|
||||
confirm,
|
||||
dryRun: explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
||||
};
|
||||
}
|
||||
if (actionRaw === "cleanup-obsolete") {
|
||||
if (lane === "v02") throw new Error("secret cleanup-obsolete is only for v0.3+ lanes after deprecated v0.3 HWPOD DB SecretRef cleanup");
|
||||
if (key !== undefined) throw new Error("secret cleanup-obsolete does not accept --key");
|
||||
if (name !== spec.obsoleteHwpodDbSecret) throw new Error(`secret cleanup-obsolete for --lane ${lane} currently only targets obsolete ${spec.obsoleteHwpodDbSecret}`);
|
||||
return {
|
||||
action: actionRaw,
|
||||
node,
|
||||
lane,
|
||||
name,
|
||||
preset: "obsolete-secret-cleanup",
|
||||
confirm,
|
||||
dryRun: explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
||||
};
|
||||
}
|
||||
if (name === spec.masterAdminApiKeySecret) {
|
||||
if (key !== undefined && key !== MASTER_ADMIN_API_KEY_KEY) throw new Error(`secret ${name} supports only key ${MASTER_ADMIN_API_KEY_KEY}`);
|
||||
return {
|
||||
action: actionRaw,
|
||||
node,
|
||||
lane,
|
||||
name,
|
||||
key: key ?? MASTER_ADMIN_API_KEY_KEY,
|
||||
preset: "master-server-admin-api-key",
|
||||
confirm,
|
||||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
||||
};
|
||||
}
|
||||
if (name === spec.bootstrapAdminSecret) {
|
||||
if (key !== undefined && key !== BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY) throw new Error(`secret ${name} supports only key ${BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY}`);
|
||||
const force = args.includes("--force");
|
||||
if (force && actionRaw !== "ensure") throw new Error("secret --force is only supported with ensure");
|
||||
return {
|
||||
action: actionRaw,
|
||||
node,
|
||||
lane,
|
||||
name,
|
||||
key: key ?? BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY,
|
||||
preset: "bootstrap-admin",
|
||||
confirm,
|
||||
force,
|
||||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
||||
};
|
||||
}
|
||||
if (name === spec.codeAgentProviderSecret) {
|
||||
if (key !== undefined && key !== CODE_AGENT_PROVIDER_OPENAI_KEY && key !== CODE_AGENT_PROVIDER_OPENCODE_KEY) {
|
||||
throw new Error(`secret ${name} supports keys ${CODE_AGENT_PROVIDER_OPENAI_KEY} and ${CODE_AGENT_PROVIDER_OPENCODE_KEY}`);
|
||||
}
|
||||
return {
|
||||
action: actionRaw,
|
||||
node,
|
||||
lane,
|
||||
name,
|
||||
key,
|
||||
preset: "code-agent-provider",
|
||||
confirm,
|
||||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
||||
};
|
||||
}
|
||||
if (name === spec.cloudApiDbSecret) {
|
||||
if (key !== undefined && key !== spec.cloudApiDbKey) throw new Error(`secret ${name} supports only key ${spec.cloudApiDbKey}`);
|
||||
if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) {
|
||||
throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`);
|
||||
}
|
||||
return {
|
||||
action: actionRaw,
|
||||
node,
|
||||
lane,
|
||||
name,
|
||||
key: key ?? spec.cloudApiDbKey,
|
||||
preset: "cloud-api-db",
|
||||
confirm,
|
||||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 240, 900),
|
||||
};
|
||||
}
|
||||
if (name !== spec.openFgaSecret) {
|
||||
throw new Error(`secret status/ensure supports --name ${spec.openFgaSecret}, ${spec.masterAdminApiKeySecret}, ${spec.bootstrapAdminSecret}, ${spec.cloudApiDbSecret}, or ${spec.codeAgentProviderSecret} for --lane ${lane}; obsolete ${spec.obsoleteHwpodDbSecret} must use cleanup-obsolete`);
|
||||
}
|
||||
if (key !== undefined && key !== OPENFGA_AUTHN_KEY && key !== OPENFGA_DATASTORE_URI_KEY && key !== OPENFGA_POSTGRES_PASSWORD_KEY) {
|
||||
throw new Error(`secret ${name} supports keys ${OPENFGA_AUTHN_KEY}, ${OPENFGA_DATASTORE_URI_KEY}, and ${OPENFGA_POSTGRES_PASSWORD_KEY}`);
|
||||
}
|
||||
if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) {
|
||||
throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`);
|
||||
}
|
||||
return {
|
||||
action: actionRaw,
|
||||
node,
|
||||
lane,
|
||||
name,
|
||||
key,
|
||||
preset: "openfga",
|
||||
confirm,
|
||||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
||||
};
|
||||
}
|
||||
|
||||
export function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecretSpec {
|
||||
const namespace = `hwlab-${input.lane}`;
|
||||
const runtimeLaneSpec = isHwlabRuntimeLane(input.lane) ? hwlabRuntimeLaneSpecForNode(input.lane, input.node) : undefined;
|
||||
const externalPostgres = runtimeLaneSpec?.externalPostgres;
|
||||
const postgresStore = runtimeLaneSpec?.runtimeStore?.postgres;
|
||||
const bootstrapAdmin = runtimeLaneSpec?.bootstrapAdmin;
|
||||
const platformDb = externalPostgres !== undefined || postgresStore?.mode === "platform-service";
|
||||
const localPostgresService = postgresStore?.serviceName ?? `${namespace}-postgres`;
|
||||
const platformPostgresService = externalPostgres?.serviceName ?? postgresStore?.serviceName ?? "g14-platform-postgres";
|
||||
const legacyPostgresHost = `${localPostgresService}.${namespace}.svc.cluster.local`;
|
||||
const platformPostgresHost = `${platformPostgresService}.${namespace}.svc.cluster.local`;
|
||||
const platformPostgresEndpointSlice = `${platformPostgresService}-host`;
|
||||
const openFgaDbName = externalPostgres?.database ?? postgresStore?.openfga?.database ?? (platformDb ? `openfga_${input.lane}` : "hwlab_openfga");
|
||||
const openFgaDbUser = externalPostgres?.openfga.role ?? postgresStore?.openfga?.role ?? (platformDb ? `openfga_${input.lane}_app` : "hwlab_openfga");
|
||||
const cloudApiDbName = externalPostgres?.database ?? postgresStore?.cloudApi?.database ?? `hwlab_${input.lane}`;
|
||||
const cloudApiDbUser = externalPostgres?.cloudApi.role ?? postgresStore?.cloudApi?.role ?? (platformDb ? `hwlab_${input.lane}_app` : `hwlab_${input.lane}`);
|
||||
return {
|
||||
node: input.node,
|
||||
lane: input.lane,
|
||||
namespace,
|
||||
platformDb,
|
||||
...(runtimeLaneSpec === undefined ? {} : { runtimeLaneSpec }),
|
||||
...(externalPostgres === undefined ? {} : { externalPostgres }),
|
||||
platformPostgresService,
|
||||
platformPostgresEndpointAddress: externalPostgres?.endpointAddress,
|
||||
platformPostgresEndpointSlice,
|
||||
postgresSecret: postgresStore?.secretName ?? `${namespace}-postgres`,
|
||||
postgresStatefulSet: postgresStore?.statefulSet ?? postgresStore?.secretName ?? `${namespace}-postgres`,
|
||||
postgresAdminUser: postgresStore?.adminUser ?? `hwlab_${input.lane}`,
|
||||
openFgaSecret: externalPostgres?.openfga.secretName ?? postgresStore?.openfga?.secretName ?? `${namespace}-openfga`,
|
||||
openFgaDbName,
|
||||
openFgaDbUser,
|
||||
openFgaDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
|
||||
masterAdminApiKeySecret: `${namespace}-master-server-admin-api-key`,
|
||||
bootstrapAdminSecret: bootstrapAdmin?.secretName ?? `${namespace}-bootstrap-admin`,
|
||||
bootstrapAdminPasswordHashKey: bootstrapAdmin?.secretKey ?? BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY,
|
||||
bootstrapAdminUsername: bootstrapAdmin?.username ?? "admin",
|
||||
bootstrapAdminDisplayName: bootstrapAdmin?.displayName ?? `HWLAB ${input.lane} Admin`,
|
||||
...(bootstrapAdmin?.passwordSourceRef === undefined ? {} : { bootstrapAdminPasswordSourceRef: bootstrapAdmin.passwordSourceRef }),
|
||||
...(bootstrapAdmin?.passwordSourceKey === undefined ? {} : { bootstrapAdminPasswordSourceKey: bootstrapAdmin.passwordSourceKey }),
|
||||
...(bootstrapAdmin?.passwordHashTransform === undefined ? {} : { bootstrapAdminPasswordHashTransform: bootstrapAdmin.passwordHashTransform }),
|
||||
bootstrapAdminSourceNamespace: BOOTSTRAP_ADMIN_SOURCE_NAMESPACE,
|
||||
bootstrapAdminSourceSecret: BOOTSTRAP_ADMIN_SOURCE_SECRET,
|
||||
cloudApiDbSecret: externalPostgres?.cloudApi.secretName ?? postgresStore?.cloudApi?.secretName ?? `hwlab-cloud-api-${input.lane}-db`,
|
||||
cloudApiDbKey: externalPostgres?.cloudApi.secretKey ?? postgresStore?.cloudApi?.secretKey ?? CLOUD_API_DB_KEY,
|
||||
cloudApiDbName,
|
||||
cloudApiDbUser,
|
||||
cloudApiDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
|
||||
cloudApiDeployment: "hwlab-cloud-api",
|
||||
obsoleteHwpodDbSecret: `hwpod-${input.lane}-db`,
|
||||
obsoleteHwpodDbName: `hwpod_${input.lane}`,
|
||||
obsoleteHwpodDbUser: `hwpod_${input.lane}_app`,
|
||||
codeAgentProviderSecret: `${namespace}-code-agent-provider`,
|
||||
codeAgentProviderSourceNamespace: CODE_AGENT_PROVIDER_SOURCE_NAMESPACE,
|
||||
codeAgentProviderSourceSecret: CODE_AGENT_PROVIDER_SOURCE_SECRET,
|
||||
fieldManager: `unidesk-hwlab-node-${input.lane}-secret`,
|
||||
};
|
||||
}
|
||||
|
||||
export function runNodeSecret(options: NodeSecretOptions): Record<string, unknown> {
|
||||
const spec = runtimeSecretSpec(options);
|
||||
if (options.preset === "obsolete-secret-cleanup") return runObsoleteSecretCleanup(options, spec);
|
||||
if (spec.externalPostgres !== undefined && options.action === "ensure" && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
|
||||
return runExternalPostgresSecretEnsure(options, spec);
|
||||
}
|
||||
const bootstrapAdminMaterial = options.preset === "bootstrap-admin" ? readBootstrapAdminSecretMaterial(spec) : null;
|
||||
const input = options.preset === "master-server-admin-api-key" && options.action === "ensure" && !options.dryRun
|
||||
? readMasterAdminApiKey(spec).key
|
||||
: options.preset === "bootstrap-admin" && options.action === "ensure" && !options.dryRun && bootstrapAdminMaterial?.ok === true
|
||||
? bootstrapAdminMaterial.passwordHash ?? ""
|
||||
: "";
|
||||
const script = options.preset === "openfga"
|
||||
? spec.platformDb ? platformDbSecretStatusScript(options, spec) : openFgaSecretScript(options, spec)
|
||||
: options.preset === "master-server-admin-api-key"
|
||||
? masterAdminApiKeySecretScript(options, spec)
|
||||
: options.preset === "bootstrap-admin"
|
||||
? bootstrapAdminSecretScript(options, spec, bootstrapAdminMaterial)
|
||||
: options.preset === "cloud-api-db"
|
||||
? spec.platformDb ? platformDbSecretStatusScript(options, spec) : cloudApiDbSecretScript(options, spec)
|
||||
: options.preset === "owned-postgres-cleanup"
|
||||
? ownedPostgresCleanupScript(options, spec)
|
||||
: codeAgentProviderSecretScript(options, spec);
|
||||
const result = runTransScript(options.node, script, input, options.timeoutSeconds);
|
||||
const status = secretStatusFromText(statusText(result), result.exitCode === 0, result.exitCode, result.stderr, spec);
|
||||
const dryRunOk = options.action === "ensure" && options.dryRun && result.exitCode === 0;
|
||||
const cleanupDryRunOk = options.action === "cleanup-owned-postgres" && options.dryRun && result.exitCode === 0;
|
||||
const obsoleteCleanupDryRunOk = options.action === "cleanup-obsolete" && options.dryRun && status.ok === true;
|
||||
const ok = dryRunOk || cleanupDryRunOk || obsoleteCleanupDryRunOk ? true : status.ok === true;
|
||||
return {
|
||||
ok,
|
||||
command: `hwlab nodes secret ${options.action}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
namespace: spec.namespace,
|
||||
secret: options.name,
|
||||
key: options.key ?? null,
|
||||
preset: options.preset,
|
||||
mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "cleanup-owned-postgres" || options.action === "cleanup-obsolete" ? "confirmed-delete" : "confirmed-ensure",
|
||||
status,
|
||||
mutation: status.mutation === true,
|
||||
result: compactCommandResult(result),
|
||||
valuesRedacted: true,
|
||||
next: ok && options.action === "status" ? undefined : nextSecretCommand(options, spec),
|
||||
};
|
||||
}
|
||||
|
||||
export function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, string> {
|
||||
if (options.action === "cleanup-owned-postgres") {
|
||||
return { ensure: `bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node ${options.node} --lane ${options.lane} --confirm` };
|
||||
}
|
||||
if (options.action === "cleanup-obsolete") {
|
||||
return { cleanup: `bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node ${options.node} --lane ${options.lane} --name ${options.name} --confirm` };
|
||||
}
|
||||
if (spec.externalPostgres !== undefined && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
|
||||
return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` };
|
||||
}
|
||||
if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
|
||||
return {
|
||||
status: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""}`,
|
||||
controlPlaneStatus: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`,
|
||||
};
|
||||
}
|
||||
return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` };
|
||||
}
|
||||
|
||||
export function runExternalPostgresSecretEnsure(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, unknown> {
|
||||
const runtimeLaneSpec = spec.runtimeLaneSpec;
|
||||
if (runtimeLaneSpec === undefined) {
|
||||
return {
|
||||
ok: false,
|
||||
command: `hwlab nodes secret ${options.action}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
namespace: spec.namespace,
|
||||
secret: options.name,
|
||||
key: options.key ?? null,
|
||||
preset: options.preset,
|
||||
mode: options.dryRun ? "dry-run" : "confirmed-ensure",
|
||||
mutation: false,
|
||||
degradedReason: "external-postgres-runtime-lane-spec-missing",
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
const sync = syncNodeExternalPostgresSecrets(runtimeLaneSpec, options.dryRun, options.timeoutSeconds);
|
||||
const shouldReadStatus = sync !== null && sync.ok === true;
|
||||
const statusResult = shouldReadStatus ? runTransScript(options.node, platformDbSecretStatusScript({ ...options, action: "status", dryRun: true }, spec), "", options.timeoutSeconds) : null;
|
||||
const status = statusResult === null
|
||||
? null
|
||||
: secretStatusFromText(statusText(statusResult), statusResult.exitCode === 0, statusResult.exitCode, statusResult.stderr, spec);
|
||||
const ok = sync !== null && sync.ok === true && (options.dryRun || status?.ok === true);
|
||||
return {
|
||||
ok,
|
||||
command: `hwlab nodes secret ${options.action}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
namespace: spec.namespace,
|
||||
secret: options.name,
|
||||
key: options.key ?? null,
|
||||
preset: options.preset,
|
||||
mode: options.dryRun ? "dry-run" : "confirmed-ensure",
|
||||
status: status ?? sync,
|
||||
externalPostgresSecretSync: sync,
|
||||
mutation: sync?.mutation === true,
|
||||
result: statusResult === null ? null : compactCommandResult(statusResult),
|
||||
valuesRedacted: true,
|
||||
next: ok ? undefined : nextSecretCommand(options, spec),
|
||||
};
|
||||
}
|
||||
|
||||
export function publicExposureSummary(exposure: HwlabRuntimePublicExposureSpec): Record<string, unknown> {
|
||||
return {
|
||||
mode: exposure.mode,
|
||||
publicBaseUrl: exposure.publicBaseUrl,
|
||||
hostname: exposure.hostname,
|
||||
expectedA: exposure.expectedA,
|
||||
frpc: {
|
||||
serverAddr: exposure.serverAddr,
|
||||
serverPort: exposure.serverPort,
|
||||
tokenSourceRef: exposure.tokenSourceRef,
|
||||
tokenSourceKey: exposure.tokenSourceKey,
|
||||
secretName: exposure.secretName,
|
||||
secretKey: exposure.secretKey,
|
||||
tokenKey: exposure.tokenKey,
|
||||
webProxy: exposure.webProxy,
|
||||
apiProxy: exposure.apiProxy,
|
||||
},
|
||||
caddy: {
|
||||
route: exposure.caddyRoute,
|
||||
configPath: exposure.caddyConfigPath,
|
||||
serviceName: exposure.caddyServiceName,
|
||||
tls: exposure.caddyTls,
|
||||
responseHeaderTimeoutSeconds: exposure.responseHeaderTimeoutSeconds,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function runNodePublicExposure(options: NodePublicExposureOptions): Record<string, unknown> {
|
||||
const exposure = options.spec.publicExposure;
|
||||
if (exposure === null || !exposure.enabled) {
|
||||
return {
|
||||
ok: false,
|
||||
command: "hwlab nodes control-plane public-exposure",
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
error: "publicExposure is not configured for this node/lane target",
|
||||
mutation: false,
|
||||
};
|
||||
}
|
||||
const source = readPublicExposureTokenSource(exposure);
|
||||
if (!source.ok) {
|
||||
return {
|
||||
ok: false,
|
||||
command: "hwlab nodes control-plane public-exposure",
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
mode: options.dryRun ? "dry-run" : "confirmed",
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
publicExposure: publicExposureSummary(exposure),
|
||||
source,
|
||||
mutation: false,
|
||||
valuesRedacted: true,
|
||||
next: { fixSecretSource: `create .state/secrets/${exposure.tokenSourceRef} with ${exposure.tokenSourceKey}=<redacted>` },
|
||||
};
|
||||
}
|
||||
const secretApplyResult = runTransScript(options.node, publicExposureSecretScript(options, exposure), source.value ?? "", options.timeoutSeconds);
|
||||
let secretFields = keyValueLinesFromText(statusText(secretApplyResult));
|
||||
let secretResult = secretApplyResult;
|
||||
if (!options.dryRun && secretApplyResult.exitCode === 0 && secretFields.afterSecretExists === "yes") {
|
||||
const restartResult = runPublicExposureFrpcRecreate(options);
|
||||
secretFields = { ...secretFields, ...keyValueLinesFromText(statusText(restartResult)) };
|
||||
secretResult = combinePublicExposureCommandResults(secretApplyResult, restartResult);
|
||||
}
|
||||
const caddyResult = runTransHostScript(exposure.caddyRoute, publicExposureCaddyScript(options, exposure), "", options.timeoutSeconds);
|
||||
const caddyFields = keyValueLinesFromText(statusText(caddyResult));
|
||||
const secretStatus = publicExposureSecretStatus(secretFields, secretResult);
|
||||
const caddyStatus = publicExposureCaddyStatus(caddyFields, caddyResult);
|
||||
const ok = secretStatus.ok === true && caddyStatus.ok === true;
|
||||
return {
|
||||
ok,
|
||||
command: "hwlab nodes control-plane public-exposure",
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
mode: options.dryRun ? "dry-run" : "confirmed",
|
||||
mutation: !options.dryRun && (secretFields.mutation === "true" || caddyFields.mutation === "true"),
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
publicExposure: publicExposureSummary(exposure),
|
||||
source: {
|
||||
ok: source.ok,
|
||||
path: source.path,
|
||||
key: exposure.tokenSourceKey,
|
||||
bytes: source.value?.length ?? 0,
|
||||
fingerprint: source.fingerprint,
|
||||
valueRedacted: true,
|
||||
},
|
||||
secret: secretStatus,
|
||||
caddy: caddyStatus,
|
||||
valuesRedacted: true,
|
||||
next: ok
|
||||
? {
|
||||
triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${options.node} --lane ${options.lane} --confirm`,
|
||||
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`,
|
||||
}
|
||||
: { retry: `bun scripts/cli.ts hwlab nodes control-plane public-exposure --node ${options.node} --lane ${options.lane} --confirm` },
|
||||
};
|
||||
}
|
||||
|
||||
export function readPublicExposureTokenSource(exposure: HwlabRuntimePublicExposureSpec): { ok: boolean; path: string; checkedPaths: string[]; key: string; value: string | null; fingerprint: string | null; error?: string } {
|
||||
const checkedPaths = publicExposureTokenSourcePaths(exposure);
|
||||
const path = checkedPaths.find((candidate) => existsSync(candidate)) ?? checkedPaths[0] ?? join(repoRoot, ".state", "secrets", exposure.tokenSourceRef);
|
||||
if (!existsSync(path)) return { ok: false, path, checkedPaths, key: exposure.tokenSourceKey, value: null, fingerprint: null, error: "secret-source-missing" };
|
||||
const values = parseEnvFile(readFileSync(path, "utf8"));
|
||||
const value = values[exposure.tokenSourceKey];
|
||||
if (value === undefined || value.length === 0) return { ok: false, path, checkedPaths, key: exposure.tokenSourceKey, value: null, fingerprint: null, error: "secret-key-missing" };
|
||||
return {
|
||||
ok: true,
|
||||
path,
|
||||
checkedPaths,
|
||||
key: exposure.tokenSourceKey,
|
||||
value,
|
||||
fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`,
|
||||
};
|
||||
}
|
||||
|
||||
export function publicExposureTokenSourcePaths(exposure: HwlabRuntimePublicExposureSpec): string[] {
|
||||
const paths = [join(repoRoot, ".state", "secrets", exposure.tokenSourceRef)];
|
||||
const marker = "/.worktree/";
|
||||
const index = repoRoot.indexOf(marker);
|
||||
if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", exposure.tokenSourceRef));
|
||||
return [...new Set(paths)];
|
||||
}
|
||||
|
||||
export function publicExposureSecretStatus(fields: Record<string, string>, result: CommandResult): Record<string, unknown> {
|
||||
const dryRun = fields.dryRun === "true";
|
||||
return {
|
||||
ok: result.exitCode === 0 && (dryRun || (
|
||||
fields.afterSecretExists === "yes"
|
||||
&& fields.strategyPatchExitCode === "0"
|
||||
&& fields.rolloutRestartExitCode === "0"
|
||||
&& fields.rolloutStatusExitCode === "0"
|
||||
)),
|
||||
dryRun,
|
||||
mutation: fields.mutation === "true",
|
||||
namespace: fields.namespace || null,
|
||||
secret: fields.secret || null,
|
||||
deployment: fields.deployment || null,
|
||||
beforeExists: fields.beforeSecretExists === "yes",
|
||||
afterExists: fields.afterSecretExists === "yes",
|
||||
tokenBytes: numericField(fields.tokenBytes),
|
||||
applyExitCode: numericField(fields.applyExitCode),
|
||||
restartMode: fields.restartMode || null,
|
||||
strategyPatchExitCode: numericField(fields.strategyPatchExitCode),
|
||||
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
|
||||
scaleDownExitCode: numericField(fields.scaleDownExitCode),
|
||||
scaleDownWaitExitCode: numericField(fields.scaleDownWaitExitCode),
|
||||
scaleDownWaitPodCount: numericField(fields.scaleDownWaitPodCount),
|
||||
scaleUpExitCode: numericField(fields.scaleUpExitCode),
|
||||
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
|
||||
readyReplicas: numericField(fields.readyReplicas),
|
||||
availableReplicas: numericField(fields.availableReplicas),
|
||||
desiredReplicas: numericField(fields.desiredReplicas),
|
||||
strategyPatchErrorPreview: fields.strategyPatchErrorPreview || null,
|
||||
restartErrorPreview: fields.restartErrorPreview || null,
|
||||
scaleDownErrorPreview: fields.scaleDownErrorPreview || null,
|
||||
scaleUpErrorPreview: fields.scaleUpErrorPreview || null,
|
||||
readyErrorPreview: fields.readyErrorPreview || null,
|
||||
exitCode: result.exitCode,
|
||||
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
|
||||
};
|
||||
}
|
||||
|
||||
export function publicExposureCaddyStatus(fields: Record<string, string>, result: CommandResult): Record<string, unknown> {
|
||||
const dryRun = fields.dryRun === "true";
|
||||
return {
|
||||
ok: result.exitCode === 0
|
||||
&& fields.afterBlockPresent === "yes"
|
||||
&& fields.validateExitCode === "0"
|
||||
&& (dryRun || (
|
||||
fields.active === "active"
|
||||
&& fields.localWebStatus === "200"
|
||||
&& fields.localApiStatus === "200"
|
||||
)),
|
||||
dryRun,
|
||||
mutation: fields.mutation === "true",
|
||||
hostname: fields.hostname || null,
|
||||
configPath: fields.configPath || null,
|
||||
beforeBlockPresent: fields.beforeBlockPresent === "yes",
|
||||
afterBlockPresent: fields.afterBlockPresent === "yes",
|
||||
staleBlocksRemoved: numericField(fields.staleBlocksRemoved),
|
||||
pythonExitCode: numericField(fields.pythonExitCode),
|
||||
validateMode: fields.validateMode || null,
|
||||
validateExitCode: numericField(fields.validateExitCode),
|
||||
reloadExitCode: numericField(fields.reloadExitCode),
|
||||
active: fields.active || null,
|
||||
localWebStatus: fields.localWebStatus || null,
|
||||
localApiStatus: fields.localApiStatus || null,
|
||||
validateErrorPreview: fields.validateErrorPreview || null,
|
||||
localWebErrorPreview: fields.localWebErrorPreview || null,
|
||||
localApiErrorPreview: fields.localApiErrorPreview || null,
|
||||
exitCode: result.exitCode,
|
||||
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
|
||||
};
|
||||
}
|
||||
|
||||
export function publicExposureSecretScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string {
|
||||
const deployment = `${options.spec.runtimeNamespace}-frpc`;
|
||||
return [
|
||||
"set +e",
|
||||
`namespace=${shellQuote(options.spec.runtimeNamespace)}`,
|
||||
`secret=${shellQuote(exposure.secretName)}`,
|
||||
`deployment=${shellQuote(deployment)}`,
|
||||
`token_key=${shellQuote(exposure.tokenKey)}`,
|
||||
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
||||
"token=$(cat)",
|
||||
"before_secret_exists=no",
|
||||
"kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && before_secret_exists=yes",
|
||||
"token_bytes=$(printf '%s' \"$token\" | wc -c | tr -d ' ')",
|
||||
"apply_exit=",
|
||||
"restart_mode=recreate-rollout-restart",
|
||||
"strategy_patch_exit=",
|
||||
"rollout_restart_exit=",
|
||||
"scale_down_exit=",
|
||||
"scale_down_wait_exit=",
|
||||
"scale_down_wait_pod_count=",
|
||||
"scale_up_exit=",
|
||||
"rollout_status_exit=",
|
||||
"ready_replicas=",
|
||||
"available_replicas=",
|
||||
"desired_replicas=",
|
||||
"mutation=false",
|
||||
"if [ \"$dry_run\" = false ]; then",
|
||||
" tmp=$(mktemp /tmp/hwlab-public-frpc-secret.XXXXXX.yaml)",
|
||||
" token_b64=$(printf '%s' \"$token\" | base64 | tr -d '\\n')",
|
||||
" cat >\"$tmp\" <<EOF",
|
||||
"apiVersion: v1",
|
||||
"kind: Secret",
|
||||
"metadata:",
|
||||
" name: $secret",
|
||||
" namespace: $namespace",
|
||||
" labels:",
|
||||
" app.kubernetes.io/part-of: hwlab",
|
||||
" app.kubernetes.io/managed-by: unidesk",
|
||||
"type: Opaque",
|
||||
"data:",
|
||||
" $token_key: $token_b64",
|
||||
"EOF",
|
||||
" kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-public-exposure -f \"$tmp\" >/tmp/hwlab-public-frpc-secret.apply.out 2>/tmp/hwlab-public-frpc-secret.apply.err",
|
||||
" apply_exit=$?",
|
||||
" rm -f \"$tmp\"",
|
||||
" if [ \"$apply_exit\" = 0 ]; then mutation=true; fi",
|
||||
"fi",
|
||||
"after_secret_exists=no",
|
||||
"kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && after_secret_exists=yes",
|
||||
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
||||
"printf 'secret\\t%s\\n' \"$secret\"",
|
||||
"printf 'deployment\\t%s\\n' \"$deployment\"",
|
||||
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
||||
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
||||
"printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
|
||||
"printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
|
||||
"printf 'tokenBytes\\t%s\\n' \"$token_bytes\"",
|
||||
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
||||
"printf 'restartMode\\t%s\\n' \"$restart_mode\"",
|
||||
"printf 'strategyPatchExitCode\\t%s\\n' \"$strategy_patch_exit\"",
|
||||
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
|
||||
"printf 'scaleDownExitCode\\t%s\\n' \"$scale_down_exit\"",
|
||||
"printf 'scaleDownWaitExitCode\\t%s\\n' \"$scale_down_wait_exit\"",
|
||||
"printf 'scaleDownWaitPodCount\\t%s\\n' \"$scale_down_wait_pod_count\"",
|
||||
"printf 'scaleUpExitCode\\t%s\\n' \"$scale_up_exit\"",
|
||||
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
|
||||
"printf 'readyReplicas\\t%s\\n' \"$ready_replicas\"",
|
||||
"printf 'availableReplicas\\t%s\\n' \"$available_replicas\"",
|
||||
"printf 'desiredReplicas\\t%s\\n' \"$desired_replicas\"",
|
||||
"if [ \"$dry_run\" = true ]; then exit 0; fi",
|
||||
"[ \"$after_secret_exists\" = yes ] && [ \"$apply_exit\" = 0 ]",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function runPublicExposureFrpcRecreate(options: NodePublicExposureOptions): CommandResult {
|
||||
const namespace = options.spec.runtimeNamespace;
|
||||
const deployment = `${namespace}-frpc`;
|
||||
const fields: Record<string, string> = {
|
||||
deployment,
|
||||
restartMode: "recreate-rollout-restart",
|
||||
strategyPatchExitCode: "",
|
||||
rolloutRestartExitCode: "",
|
||||
scaleDownExitCode: "",
|
||||
scaleDownWaitExitCode: "",
|
||||
scaleDownWaitPodCount: "",
|
||||
scaleUpExitCode: "",
|
||||
rolloutStatusExitCode: "",
|
||||
readyReplicas: "",
|
||||
availableReplicas: "",
|
||||
desiredReplicas: "",
|
||||
strategyPatchErrorPreview: "",
|
||||
restartErrorPreview: "",
|
||||
scaleDownErrorPreview: "",
|
||||
scaleUpErrorPreview: "",
|
||||
readyErrorPreview: "",
|
||||
};
|
||||
const stdoutParts: string[] = [];
|
||||
const stderrParts: string[] = [];
|
||||
const addResult = (result: CommandResult): void => {
|
||||
if (result.stdout.trim()) stdoutParts.push(result.stdout.trim());
|
||||
if (result.stderr.trim()) stderrParts.push(result.stderr.trim());
|
||||
};
|
||||
const shortTimeout = Math.min(Math.max(20, options.timeoutSeconds), 55);
|
||||
const strategyPatch = runTransScript(options.node, [
|
||||
"set +e",
|
||||
`namespace=${shellQuote(namespace)}`,
|
||||
`deployment=${shellQuote(deployment)}`,
|
||||
"kubectl -n \"$namespace\" patch deploy/\"$deployment\" --type=merge -p '{\"spec\":{\"strategy\":{\"type\":\"Recreate\",\"rollingUpdate\":null}}}' >/tmp/hwlab-public-frpc-strategy-patch.out 2>/tmp/hwlab-public-frpc-strategy-patch.err",
|
||||
"code=$?",
|
||||
"printf 'strategyPatchExitCode\\t%s\\n' \"$code\"",
|
||||
"printf 'strategyPatchErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-strategy-patch.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-strategy-patch.err | cut -c1-1000 || true)\"",
|
||||
"exit \"$code\"",
|
||||
].join("\n"), "", shortTimeout);
|
||||
addResult(strategyPatch);
|
||||
Object.assign(fields, keyValueLinesFromText(statusText(strategyPatch)));
|
||||
|
||||
if (strategyPatch.exitCode === 0) {
|
||||
const restart = runTransScript(options.node, [
|
||||
"set +e",
|
||||
`namespace=${shellQuote(namespace)}`,
|
||||
`deployment=${shellQuote(deployment)}`,
|
||||
"kubectl -n \"$namespace\" rollout restart deploy/\"$deployment\" >/tmp/hwlab-public-frpc-restart.out 2>/tmp/hwlab-public-frpc-restart.err",
|
||||
"code=$?",
|
||||
"printf 'rolloutRestartExitCode\\t%s\\n' \"$code\"",
|
||||
"printf 'restartErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-restart.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-restart.err | cut -c1-1000 || true)\"",
|
||||
"exit \"$code\"",
|
||||
].join("\n"), "", shortTimeout);
|
||||
addResult(restart);
|
||||
Object.assign(fields, keyValueLinesFromText(statusText(restart)));
|
||||
}
|
||||
|
||||
if (fields.rolloutRestartExitCode === "0") {
|
||||
const deadline = Date.now() + 55_000;
|
||||
while (Date.now() <= deadline) {
|
||||
const probe = runTransScript(options.node, [
|
||||
"set +e",
|
||||
`namespace=${shellQuote(namespace)}`,
|
||||
`deployment=${shellQuote(deployment)}`,
|
||||
"kubectl -n \"$namespace\" get deploy \"$deployment\" -o jsonpath='{.spec.replicas}{\"\\t\"}{.status.readyReplicas}{\"\\t\"}{.status.availableReplicas}' >/tmp/hwlab-public-frpc-ready.out 2>/tmp/hwlab-public-frpc-ready.err",
|
||||
"code=$?",
|
||||
"values=$(cat /tmp/hwlab-public-frpc-ready.out 2>/dev/null || true)",
|
||||
"desired=$(printf '%s' \"$values\" | cut -f1)",
|
||||
"ready=$(printf '%s' \"$values\" | cut -f2)",
|
||||
"available=$(printf '%s' \"$values\" | cut -f3)",
|
||||
"printf 'desiredReplicas\\t%s\\n' \"$desired\"",
|
||||
"printf 'readyReplicas\\t%s\\n' \"$ready\"",
|
||||
"printf 'availableReplicas\\t%s\\n' \"$available\"",
|
||||
"printf 'readyErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-ready.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-ready.err | cut -c1-1000 || true)\"",
|
||||
"[ \"$code\" = 0 ] && [ \"$desired\" = 1 ] && [ \"$ready\" = 1 ] && [ \"$available\" = 1 ]",
|
||||
].join("\n"), "", shortTimeout);
|
||||
addResult(probe);
|
||||
Object.assign(fields, keyValueLinesFromText(statusText(probe)));
|
||||
if (probe.exitCode === 0) {
|
||||
fields.rolloutStatusExitCode = "0";
|
||||
break;
|
||||
}
|
||||
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
||||
}
|
||||
}
|
||||
if (fields.rolloutStatusExitCode === "") fields.rolloutStatusExitCode = "1";
|
||||
|
||||
const ok = fields.strategyPatchExitCode === "0"
|
||||
&& fields.rolloutRestartExitCode === "0"
|
||||
&& fields.rolloutStatusExitCode === "0";
|
||||
const stdout = Object.entries(fields).map(([key, value]) => `${key}\t${value}`).join("\n") + "\n";
|
||||
return {
|
||||
command: [transPath(), `${options.node}:k3s`, "sh", "--", "<public-exposure-frpc-recreate>"],
|
||||
cwd: repoRoot,
|
||||
exitCode: ok ? 0 : 1,
|
||||
stdout: [stdout.trim(), ...stdoutParts].filter(Boolean).join("\n") + "\n",
|
||||
stderr: stderrParts.join("\n"),
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function combinePublicExposureCommandResults(first: CommandResult, second: CommandResult): CommandResult {
|
||||
return {
|
||||
command: [...first.command, "&&", ...second.command],
|
||||
cwd: repoRoot,
|
||||
exitCode: first.exitCode === 0 && second.exitCode === 0 ? 0 : second.exitCode ?? first.exitCode,
|
||||
stdout: [first.stdout.trim(), second.stdout.trim()].filter(Boolean).join("\n") + "\n",
|
||||
stderr: [first.stderr.trim(), second.stderr.trim()].filter(Boolean).join("\n"),
|
||||
signal: second.signal ?? first.signal,
|
||||
timedOut: first.timedOut || second.timedOut,
|
||||
};
|
||||
}
|
||||
|
||||
export function publicExposureCaddyScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string {
|
||||
const blockB64 = Buffer.from(publicExposureCaddyBlock(exposure), "utf8").toString("base64");
|
||||
const marker = `unidesk managed ${exposure.hostname}`;
|
||||
return [
|
||||
"set +e",
|
||||
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
||||
`hostname=${shellQuote(exposure.hostname)}`,
|
||||
`config_path=${shellQuote(exposure.caddyConfigPath)}`,
|
||||
`service=${shellQuote(exposure.caddyServiceName)}`,
|
||||
`marker=${shellQuote(marker)}`,
|
||||
`block_b64=${shellQuote(blockB64)}`,
|
||||
`expected_a=${shellQuote(exposure.expectedA)}`,
|
||||
`web_port=${shellQuote(String(exposure.webProxy.remotePort))}`,
|
||||
`api_port=${shellQuote(String(exposure.apiProxy.remotePort))}`,
|
||||
"rm -f /tmp/hwlab-public-caddy-validate.out /tmp/hwlab-public-caddy-validate.err /tmp/hwlab-public-caddy-python.err /tmp/hwlab-public-caddy-web.err /tmp/hwlab-public-caddy-api.err",
|
||||
"tmp=$(mktemp -d)",
|
||||
"block=\"$tmp/block\"",
|
||||
"next=\"$tmp/Caddyfile\"",
|
||||
"printf '%s' \"$block_b64\" | base64 -d >\"$block\"",
|
||||
"before_present=no",
|
||||
"if [ -f \"$config_path\" ] && grep -Fq \"# BEGIN $marker\" \"$config_path\"; then before_present=yes; fi",
|
||||
"if [ -f \"$config_path\" ]; then cp \"$config_path\" \"$next\"; else : >\"$next\"; fi",
|
||||
"stale_blocks_removed=$(python3 - \"$next\" \"$block\" \"$marker\" \"$web_port\" \"$api_port\" 2>/tmp/hwlab-public-caddy-python.err <<'PY'",
|
||||
"import pathlib, re, sys",
|
||||
"config = pathlib.Path(sys.argv[1])",
|
||||
"block = pathlib.Path(sys.argv[2]).read_text(encoding='utf-8')",
|
||||
"marker = sys.argv[3]",
|
||||
"web_port = sys.argv[4]",
|
||||
"api_port = sys.argv[5]",
|
||||
"current_name = marker[len('unidesk managed '):] if marker.startswith('unidesk managed ') else marker",
|
||||
"text = config.read_text(encoding='utf-8') if config.exists() else ''",
|
||||
"begin = f'# BEGIN {marker}'",
|
||||
"end = f'# END {marker}'",
|
||||
"managed = f'{begin}\\n{block.rstrip()}\\n{end}\\n'",
|
||||
"stale_removed = [0]",
|
||||
"pattern = re.compile(r'(?ms)^# BEGIN unidesk managed (?P<name>[^\\n]+)\\n(?P<body>.*?)\\n# END unidesk managed (?P=name)\\n*')",
|
||||
"def keep(match):",
|
||||
" name = match.group('name')",
|
||||
" body = match.group('body')",
|
||||
" if name != current_name and (f'127.0.0.1:{web_port}' in body or f'127.0.0.1:{api_port}' in body):",
|
||||
" stale_removed[0] += 1",
|
||||
" return ''",
|
||||
" return match.group(0)",
|
||||
"text = pattern.sub(keep, text)",
|
||||
"if begin in text and end in text:",
|
||||
" start = text.index(begin)",
|
||||
" stop = text.index(end, start) + len(end)",
|
||||
" text = text[:start].rstrip() + '\\n\\n' + managed.rstrip() + '\\n' + text[stop:].lstrip('\\n')",
|
||||
"else:",
|
||||
" text = text.rstrip() + '\\n\\n' + managed",
|
||||
"config.write_text(text, encoding='utf-8')",
|
||||
"print(stale_removed[0])",
|
||||
"PY",
|
||||
")",
|
||||
"python_exit=$?",
|
||||
"validate_exit=1",
|
||||
"validate_mode=validate",
|
||||
"if [ \"$python_exit\" != 0 ]; then",
|
||||
" validate_mode=python",
|
||||
" validate_exit=$python_exit",
|
||||
"else",
|
||||
" if [ \"$dry_run\" = true ]; then",
|
||||
" validate_mode=adapt",
|
||||
" caddy adapt --config \"$next\" --adapter caddyfile >/tmp/hwlab-public-caddy-validate.out 2>/tmp/hwlab-public-caddy-validate.err",
|
||||
" else",
|
||||
" sudo caddy validate --config \"$next\" --adapter caddyfile >/tmp/hwlab-public-caddy-validate.out 2>/tmp/hwlab-public-caddy-validate.err",
|
||||
" fi",
|
||||
" validate_exit=$?",
|
||||
"fi",
|
||||
"reload_exit=",
|
||||
"mutation=false",
|
||||
"if [ \"$dry_run\" = false ] && [ \"$validate_exit\" = 0 ]; then",
|
||||
" sudo install -m 0644 \"$next\" \"$config_path\" >/tmp/hwlab-public-caddy-install.out 2>/tmp/hwlab-public-caddy-install.err",
|
||||
" install_exit=$?",
|
||||
" if [ \"$install_exit\" = 0 ]; then",
|
||||
" mutation=true",
|
||||
" sudo systemctl reload \"$service\" >/tmp/hwlab-public-caddy-reload.out 2>/tmp/hwlab-public-caddy-reload.err || sudo systemctl restart \"$service\" >>/tmp/hwlab-public-caddy-reload.out 2>>/tmp/hwlab-public-caddy-reload.err",
|
||||
" reload_exit=$?",
|
||||
" else",
|
||||
" reload_exit=$install_exit",
|
||||
" fi",
|
||||
"fi",
|
||||
"active=$(systemctl is-active \"$service\" 2>/dev/null || true)",
|
||||
"after_present=no",
|
||||
"if [ \"$dry_run\" = true ]; then grep -Fq \"# BEGIN $marker\" \"$next\" && after_present=yes; else grep -Fq \"# BEGIN $marker\" \"$config_path\" && after_present=yes; fi",
|
||||
"local_web_status=",
|
||||
"local_api_status=",
|
||||
"if [ \"$dry_run\" = false ]; then",
|
||||
" local_web_status=$(curl -kfsS --max-time 15 --resolve \"$hostname:443:127.0.0.1\" \"https://$hostname/\" -o /dev/null -w '%{http_code}' 2>/tmp/hwlab-public-caddy-web.err || true)",
|
||||
" local_api_status=$(curl -kfsS --max-time 15 --resolve \"$hostname:443:127.0.0.1\" \"https://$hostname/health/live\" -o /dev/null -w '%{http_code}' 2>/tmp/hwlab-public-caddy-api.err || true)",
|
||||
"fi",
|
||||
"validate_err_preview=$(cat /tmp/hwlab-public-caddy-python.err /tmp/hwlab-public-caddy-validate.err 2>/dev/null | tr '\\n' ';' | cut -c1-1000 || true)",
|
||||
"local_web_err_preview=$([ -f /tmp/hwlab-public-caddy-web.err ] && tr '\\n' ';' </tmp/hwlab-public-caddy-web.err 2>/dev/null | cut -c1-1000 || true)",
|
||||
"local_api_err_preview=$([ -f /tmp/hwlab-public-caddy-api.err ] && tr '\\n' ';' </tmp/hwlab-public-caddy-api.err 2>/dev/null | cut -c1-1000 || true)",
|
||||
"printf 'hostname\\t%s\\n' \"$hostname\"",
|
||||
"printf 'expectedA\\t%s\\n' \"$expected_a\"",
|
||||
"printf 'configPath\\t%s\\n' \"$config_path\"",
|
||||
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
||||
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
||||
"printf 'beforeBlockPresent\\t%s\\n' \"$before_present\"",
|
||||
"printf 'afterBlockPresent\\t%s\\n' \"$after_present\"",
|
||||
"printf 'staleBlocksRemoved\\t%s\\n' \"$stale_blocks_removed\"",
|
||||
"printf 'pythonExitCode\\t%s\\n' \"$python_exit\"",
|
||||
"printf 'validateMode\\t%s\\n' \"$validate_mode\"",
|
||||
"printf 'validateExitCode\\t%s\\n' \"$validate_exit\"",
|
||||
"printf 'reloadExitCode\\t%s\\n' \"$reload_exit\"",
|
||||
"printf 'active\\t%s\\n' \"$active\"",
|
||||
"printf 'localWebStatus\\t%s\\n' \"$local_web_status\"",
|
||||
"printf 'localApiStatus\\t%s\\n' \"$local_api_status\"",
|
||||
"printf 'validateErrorPreview\\t%s\\n' \"$validate_err_preview\"",
|
||||
"printf 'localWebErrorPreview\\t%s\\n' \"$local_web_err_preview\"",
|
||||
"printf 'localApiErrorPreview\\t%s\\n' \"$local_api_err_preview\"",
|
||||
"rm -rf \"$tmp\"",
|
||||
"[ \"$validate_exit\" = 0 ] && [ \"$after_present\" = yes ]",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function publicExposureCaddyBlock(exposure: HwlabRuntimePublicExposureSpec): string {
|
||||
const tlsLines = exposure.caddyTls === "internal" ? " tls internal\n" : "";
|
||||
return `${exposure.hostname} {
|
||||
${tlsLines} @api path /health* /auth* /v1* /json-rpc* /openapi* /docs* /swagger*
|
||||
reverse_proxy @api 127.0.0.1:${exposure.apiProxy.remotePort} {
|
||||
transport http {
|
||||
response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s
|
||||
}
|
||||
}
|
||||
reverse_proxy 127.0.0.1:${exposure.webProxy.remotePort} {
|
||||
transport http {
|
||||
response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s
|
||||
}
|
||||
}
|
||||
}`;
|
||||
}
|
||||
|
||||
export function runTransScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult {
|
||||
return runCommand([transPath(), `${node}:k3s`, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
|
||||
}
|
||||
|
||||
export function runTransHostScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult {
|
||||
return runCommand([transPath(), node, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
|
||||
}
|
||||
|
||||
export function runTransWorkspaceStdinScript(node: string, workspace: string, script: string, timeoutSeconds: number): CommandResult {
|
||||
return runCommand([transPath(), `${node}:${workspace}`, "sh"], repoRoot, { input: script, timeoutMs: timeoutSeconds * 1000 });
|
||||
}
|
||||
|
||||
export function runObsoleteSecretCleanup(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, unknown> {
|
||||
const kubernetesResult = runTransScript(options.node, obsoleteSecretCleanupScript(options, spec), "", options.timeoutSeconds);
|
||||
const kubernetesStatus = secretStatusFromText(statusText(kubernetesResult), kubernetesResult.exitCode === 0, kubernetesResult.exitCode, kubernetesResult.stderr, spec);
|
||||
const hostOptions = { ...options, dryRun: options.dryRun || kubernetesStatus.ok !== true };
|
||||
const platformDbResult = runTransHostScript(options.node, obsoletePlatformDbCleanupScript(hostOptions, spec), "", options.timeoutSeconds);
|
||||
const platformDbStatus = obsoletePlatformDbStatusFromText(statusText(platformDbResult), platformDbResult.exitCode === 0, platformDbResult.exitCode, platformDbResult.stderr, spec);
|
||||
const ok = kubernetesStatus.ok === true && platformDbStatus.ok === true && (options.dryRun || hostOptions.dryRun === false);
|
||||
const mutation = kubernetesStatus.mutation === true || platformDbStatus.mutation === true;
|
||||
return {
|
||||
ok,
|
||||
command: `hwlab nodes secret ${options.action}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
namespace: spec.namespace,
|
||||
secret: options.name,
|
||||
key: null,
|
||||
preset: options.preset,
|
||||
mode: options.dryRun ? "dry-run" : "confirmed-delete",
|
||||
status: {
|
||||
ok,
|
||||
preset: "obsolete-hwpod-db-cleanup",
|
||||
dryRun: options.dryRun,
|
||||
mutation,
|
||||
kubernetesSecret: kubernetesStatus,
|
||||
platformDatabase: platformDbStatus,
|
||||
hostMutationSkipped: !options.dryRun && hostOptions.dryRun,
|
||||
summary: ok
|
||||
? `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, and ${spec.obsoleteHwpodDbUser} are absent or ready to remove`
|
||||
: `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, or ${spec.obsoleteHwpodDbUser} still needs cleanup`,
|
||||
},
|
||||
mutation,
|
||||
result: {
|
||||
kubernetesSecret: compactCommandResult(kubernetesResult),
|
||||
platformDatabase: compactCommandResult(platformDbResult),
|
||||
},
|
||||
valuesRedacted: true,
|
||||
next: ok ? undefined : nextSecretCommand(options, spec),
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user