Codex
1062 lines
54 KiB
TypeScript
1062 lines
54 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. public-exposure module for scripts/src/hwlab-node-impl.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-node-impl.ts:11201-12084 for #903.
|
|
|
|
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
|
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand, type CommandResult } from "../command";
|
|
import { startJob } from "../jobs";
|
|
import { classifySshTcpPoolFailure } from "../ssh";
|
|
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
|
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
|
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
|
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
|
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
|
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
|
|
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
|
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
|
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
|
|
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
|
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
|
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
|
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
|
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
|
import type { RenderedCliResult } from "../output";
|
|
|
|
import type { CodeAgentProviderSecretMaterial, NodePublicExposureOptions, NodeSecretOptions, RuntimeSecretSpec } from "./entry";
|
|
import { BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY, BOOTSTRAP_ADMIN_SOURCE_NAMESPACE, BOOTSTRAP_ADMIN_SOURCE_SECRET, CLOUD_API_DB_KEY, CODE_AGENT_PROVIDER_OPENAI_KEY, CODE_AGENT_PROVIDER_OPENCODE_KEY, CODE_AGENT_PROVIDER_SOURCE_NAMESPACE, CODE_AGENT_PROVIDER_SOURCE_SECRET, MASTER_ADMIN_API_KEY_KEY, OPENFGA_AUTHN_KEY, OPENFGA_DATASTORE_URI_KEY, OPENFGA_POSTGRES_PASSWORD_KEY } from "./entry";
|
|
import { transPath } from "./runtime-common";
|
|
import { bootstrapAdminSecretScript, cloudApiDbSecretScript, codeAgentProviderSecretScript, masterAdminApiKeySecretScript, obsoletePlatformDbCleanupScript, obsoletePlatformDbStatusFromText, obsoleteSecretCleanupScript, openFgaSecretScript, ownedPostgresCleanupScript, platformDbSecretStatusScript, secretStatusFromText } from "./secret-scripts";
|
|
import { assertLane, assertNodeId, compactCommandResult, keyValueLinesFromText, numericField, optionValue, positiveIntegerOption, readMasterAdminApiKey, requiredOption, shellQuote, statusText } from "./utils";
|
|
import { displayRepoPath, localSecretSourcePaths, parseEnvFile, readBootstrapAdminSecretMaterial, shortSecretFingerprint, syncNodeExternalPostgresSecrets } from "./web-probe";
|
|
import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes";
|
|
|
|
export function isSafeWebProbeScriptRunDir(value: string | null): value is string {
|
|
return typeof value === "string"
|
|
&& value.length > 0
|
|
&& !value.includes("\0")
|
|
&& !value.includes("..")
|
|
&& /\.state\/web-probe-script\/run\.[A-Za-z0-9]+$/u.test(value);
|
|
}
|
|
|
|
export function isSafeWebProbeScriptReportPath(value: string | null): value is string {
|
|
return typeof value === "string"
|
|
&& isSafeWebProbeScriptArtifactPath(value)
|
|
&& value.endsWith("/web-probe-script-report.json");
|
|
}
|
|
|
|
export function isSafeWebProbeScriptArtifactPath(value: string): boolean {
|
|
return typeof value === "string"
|
|
&& value.length > 0
|
|
&& !value.includes("\0")
|
|
&& !value.includes("..")
|
|
&& /\.state\/web-probe-script\/run\.[A-Za-z0-9]+\/[^/]+[.](?:png|json)$/u.test(value);
|
|
}
|
|
|
|
export function parseSecretOptions(args: string[]): NodeSecretOptions {
|
|
const [actionRaw] = args;
|
|
if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "cleanup-owned-postgres" && actionRaw !== "cleanup-obsolete") {
|
|
throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key|hwlab-vNN-bootstrap-admin|hwlab-cloud-api-vNN-db|hwlab-vNN-code-agent-provider [--dry-run|--confirm] | cleanup-owned-postgres --node NODE --lane vNN [--dry-run|--confirm] | cleanup-obsolete --node NODE --lane vNN --name hwpod-vNN-db [--dry-run|--confirm]");
|
|
}
|
|
const node = requiredOption(args, "--node");
|
|
assertNodeId(node);
|
|
const lane = requiredOption(args, "--lane");
|
|
assertLane(lane);
|
|
const spec = runtimeSecretSpec({ node, lane });
|
|
const name = optionValue(args, "--name") ?? spec.openFgaSecret;
|
|
const key = optionValue(args, "--key");
|
|
const confirm = args.includes("--confirm");
|
|
const explicitDryRun = args.includes("--dry-run");
|
|
if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run");
|
|
if (actionRaw === "cleanup-owned-postgres") {
|
|
if (lane === "v02") throw new Error("secret cleanup-owned-postgres is only for v0.3+ lanes after migration to G14 platform Postgres");
|
|
if (key !== undefined) throw new Error("secret cleanup-owned-postgres does not accept --key");
|
|
if (name !== spec.openFgaSecret && name !== spec.postgresSecret) throw new Error(`secret cleanup-owned-postgres for --lane ${lane} targets ${spec.postgresSecret}; omit --name or pass --name ${spec.postgresSecret}`);
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name: spec.postgresSecret,
|
|
preset: "owned-postgres-cleanup",
|
|
confirm,
|
|
dryRun: explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
if (actionRaw === "cleanup-obsolete") {
|
|
if (lane === "v02") throw new Error("secret cleanup-obsolete is only for v0.3+ lanes after deprecated v0.3 HWPOD DB SecretRef cleanup");
|
|
if (key !== undefined) throw new Error("secret cleanup-obsolete does not accept --key");
|
|
if (name !== spec.obsoleteHwpodDbSecret) throw new Error(`secret cleanup-obsolete for --lane ${lane} currently only targets obsolete ${spec.obsoleteHwpodDbSecret}`);
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
preset: "obsolete-secret-cleanup",
|
|
confirm,
|
|
dryRun: explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
if (name === spec.masterAdminApiKeySecret) {
|
|
if (key !== undefined && key !== MASTER_ADMIN_API_KEY_KEY) throw new Error(`secret ${name} supports only key ${MASTER_ADMIN_API_KEY_KEY}`);
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
key: key ?? MASTER_ADMIN_API_KEY_KEY,
|
|
preset: "master-server-admin-api-key",
|
|
confirm,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
if (name === spec.bootstrapAdminSecret) {
|
|
if (key !== undefined && key !== BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY) throw new Error(`secret ${name} supports only key ${BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY}`);
|
|
const force = args.includes("--force");
|
|
if (force && actionRaw !== "ensure") throw new Error("secret --force is only supported with ensure");
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
key: key ?? BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY,
|
|
preset: "bootstrap-admin",
|
|
confirm,
|
|
force,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
if (name === spec.codeAgentProviderSecret) {
|
|
if (key !== undefined && key !== CODE_AGENT_PROVIDER_OPENAI_KEY && key !== CODE_AGENT_PROVIDER_OPENCODE_KEY) {
|
|
throw new Error(`secret ${name} supports keys ${CODE_AGENT_PROVIDER_OPENAI_KEY} and ${CODE_AGENT_PROVIDER_OPENCODE_KEY}`);
|
|
}
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
key,
|
|
preset: "code-agent-provider",
|
|
confirm,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
if (name === spec.cloudApiDbSecret) {
|
|
if (key !== undefined && key !== spec.cloudApiDbKey) throw new Error(`secret ${name} supports only key ${spec.cloudApiDbKey}`);
|
|
if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) {
|
|
throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`);
|
|
}
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
key: key ?? spec.cloudApiDbKey,
|
|
preset: "cloud-api-db",
|
|
confirm,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 240, 900),
|
|
};
|
|
}
|
|
if (name !== spec.openFgaSecret) {
|
|
throw new Error(`secret status/ensure supports --name ${spec.openFgaSecret}, ${spec.masterAdminApiKeySecret}, ${spec.bootstrapAdminSecret}, ${spec.cloudApiDbSecret}, or ${spec.codeAgentProviderSecret} for --lane ${lane}; obsolete ${spec.obsoleteHwpodDbSecret} must use cleanup-obsolete`);
|
|
}
|
|
if (key !== undefined && key !== OPENFGA_AUTHN_KEY && key !== OPENFGA_DATASTORE_URI_KEY && key !== OPENFGA_POSTGRES_PASSWORD_KEY) {
|
|
throw new Error(`secret ${name} supports keys ${OPENFGA_AUTHN_KEY}, ${OPENFGA_DATASTORE_URI_KEY}, and ${OPENFGA_POSTGRES_PASSWORD_KEY}`);
|
|
}
|
|
if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) {
|
|
throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`);
|
|
}
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
key,
|
|
preset: "openfga",
|
|
confirm,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
|
|
export function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecretSpec {
|
|
const namespace = `hwlab-${input.lane}`;
|
|
const runtimeLaneSpec = isHwlabRuntimeLane(input.lane) ? hwlabRuntimeLaneSpecForNode(input.lane, input.node) : undefined;
|
|
const externalPostgres = runtimeLaneSpec === undefined ? undefined : hwlabRuntimeActiveExternalPostgres(runtimeLaneSpec);
|
|
const postgresStore = runtimeLaneSpec?.runtimeStore?.postgres;
|
|
const bootstrapAdmin = runtimeLaneSpec?.bootstrapAdmin;
|
|
const platformDb = postgresStore?.mode === "platform-service";
|
|
const localPostgresService = postgresStore?.serviceName ?? `${namespace}-postgres`;
|
|
const platformPostgresService = externalPostgres?.serviceName ?? postgresStore?.serviceName ?? "g14-platform-postgres";
|
|
const platformPostgresRuntimeAccess = externalPostgres?.runtimeAccess;
|
|
const legacyPostgresHost = `${localPostgresService}.${namespace}.svc.cluster.local`;
|
|
const platformPostgresHost = `${platformPostgresService}.${namespace}.svc.cluster.local`;
|
|
const platformPostgresEndpointSlice = `${platformPostgresService}-host`;
|
|
const openFgaDbName = externalPostgres?.database ?? postgresStore?.openfga?.database ?? (platformDb ? `openfga_${input.lane}` : "hwlab_openfga");
|
|
const openFgaDbUser = externalPostgres?.openfga.role ?? postgresStore?.openfga?.role ?? (platformDb ? `openfga_${input.lane}_app` : "hwlab_openfga");
|
|
const cloudApiDbName = externalPostgres?.database ?? postgresStore?.cloudApi?.database ?? `hwlab_${input.lane}`;
|
|
const cloudApiDbUser = externalPostgres?.cloudApi.role ?? postgresStore?.cloudApi?.role ?? (platformDb ? `hwlab_${input.lane}_app` : `hwlab_${input.lane}`);
|
|
return {
|
|
node: input.node,
|
|
lane: input.lane,
|
|
namespace,
|
|
platformDb,
|
|
...(runtimeLaneSpec === undefined ? {} : { runtimeLaneSpec }),
|
|
...(externalPostgres === undefined ? {} : { externalPostgres }),
|
|
platformPostgresService,
|
|
platformPostgresEndpointAddress: platformPostgresRuntimeAccess?.endpointAddress ?? externalPostgres?.endpointAddress,
|
|
platformPostgresEndpointSlice,
|
|
postgresSecret: postgresStore?.secretName ?? `${namespace}-postgres`,
|
|
postgresStatefulSet: postgresStore?.statefulSet ?? postgresStore?.secretName ?? `${namespace}-postgres`,
|
|
postgresAdminUser: postgresStore?.adminUser ?? `hwlab_${input.lane}`,
|
|
openFgaSecret: externalPostgres?.openfga.secretName ?? postgresStore?.openfga?.secretName ?? `${namespace}-openfga`,
|
|
openFgaDbName,
|
|
openFgaDbUser,
|
|
openFgaDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
|
|
masterAdminApiKeySecret: `${namespace}-master-server-admin-api-key`,
|
|
bootstrapAdminSecret: bootstrapAdmin?.secretName ?? `${namespace}-bootstrap-admin`,
|
|
bootstrapAdminPasswordHashKey: bootstrapAdmin?.secretKey ?? BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY,
|
|
bootstrapAdminUsername: bootstrapAdmin?.username ?? "admin",
|
|
bootstrapAdminDisplayName: bootstrapAdmin?.displayName ?? `HWLAB ${input.lane} Admin`,
|
|
...(bootstrapAdmin?.usernameSourceRef === undefined ? {} : { bootstrapAdminUsernameSourceRef: bootstrapAdmin.usernameSourceRef }),
|
|
...(bootstrapAdmin?.usernameSourceKey === undefined ? {} : { bootstrapAdminUsernameSourceKey: bootstrapAdmin.usernameSourceKey }),
|
|
...(bootstrapAdmin?.usernameSourceLine === undefined ? {} : { bootstrapAdminUsernameSourceLine: bootstrapAdmin.usernameSourceLine }),
|
|
...(bootstrapAdmin?.passwordSourceRef === undefined ? {} : { bootstrapAdminPasswordSourceRef: bootstrapAdmin.passwordSourceRef }),
|
|
...(bootstrapAdmin?.passwordSourceKey === undefined ? {} : { bootstrapAdminPasswordSourceKey: bootstrapAdmin.passwordSourceKey }),
|
|
...(bootstrapAdmin?.passwordSourceLine === undefined ? {} : { bootstrapAdminPasswordSourceLine: bootstrapAdmin.passwordSourceLine }),
|
|
...(bootstrapAdmin?.passwordHashTransform === undefined ? {} : { bootstrapAdminPasswordHashTransform: bootstrapAdmin.passwordHashTransform }),
|
|
bootstrapAdminSourceNamespace: BOOTSTRAP_ADMIN_SOURCE_NAMESPACE,
|
|
bootstrapAdminSourceSecret: BOOTSTRAP_ADMIN_SOURCE_SECRET,
|
|
cloudApiDbSecret: externalPostgres?.cloudApi.secretName ?? postgresStore?.cloudApi?.secretName ?? `hwlab-cloud-api-${input.lane}-db`,
|
|
cloudApiDbKey: externalPostgres?.cloudApi.secretKey ?? postgresStore?.cloudApi?.secretKey ?? CLOUD_API_DB_KEY,
|
|
cloudApiDbName,
|
|
cloudApiDbUser,
|
|
cloudApiDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
|
|
cloudApiDeployment: "hwlab-cloud-api",
|
|
obsoleteHwpodDbSecret: `hwpod-${input.lane}-db`,
|
|
obsoleteHwpodDbName: `hwpod_${input.lane}`,
|
|
obsoleteHwpodDbUser: `hwpod_${input.lane}_app`,
|
|
codeAgentProviderSecret: runtimeLaneSpec?.codeAgentProvider?.secretName ?? `${namespace}-code-agent-provider`,
|
|
codeAgentProviderSourceNamespace: CODE_AGENT_PROVIDER_SOURCE_NAMESPACE,
|
|
codeAgentProviderSourceSecret: CODE_AGENT_PROVIDER_SOURCE_SECRET,
|
|
...(runtimeLaneSpec?.codeAgentProvider?.sourceRef === undefined ? {} : { codeAgentProviderSourceRef: runtimeLaneSpec.codeAgentProvider.sourceRef }),
|
|
...(runtimeLaneSpec?.codeAgentProvider?.openaiSourceKey === undefined ? {} : { codeAgentProviderOpenaiSourceKey: runtimeLaneSpec.codeAgentProvider.openaiSourceKey }),
|
|
...(runtimeLaneSpec?.codeAgentProvider?.opencodeSourceKey === undefined ? {} : { codeAgentProviderOpencodeSourceKey: runtimeLaneSpec.codeAgentProvider.opencodeSourceKey }),
|
|
fieldManager: `unidesk-hwlab-node-${input.lane}-secret`,
|
|
};
|
|
}
|
|
|
|
export function runNodeSecret(options: NodeSecretOptions): Record<string, unknown> {
|
|
const spec = runtimeSecretSpec(options);
|
|
if (options.preset === "obsolete-secret-cleanup") return runObsoleteSecretCleanup(options, spec);
|
|
if (spec.externalPostgres !== undefined && options.action === "ensure" && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
|
|
return runExternalPostgresSecretEnsure(options, spec);
|
|
}
|
|
const bootstrapAdminMaterial = options.preset === "bootstrap-admin" ? readBootstrapAdminSecretMaterial(spec) : null;
|
|
const codeAgentProviderMaterial = options.preset === "code-agent-provider" && spec.codeAgentProviderSourceRef !== undefined
|
|
? readCodeAgentProviderSecretMaterial(spec)
|
|
: null;
|
|
const input = options.preset === "master-server-admin-api-key" && options.action === "ensure" && !options.dryRun
|
|
? readMasterAdminApiKey(spec).key
|
|
: options.preset === "bootstrap-admin" && options.action === "ensure" && !options.dryRun && bootstrapAdminMaterial?.ok === true
|
|
? bootstrapAdminMaterial.passwordHash ?? ""
|
|
: "";
|
|
const script = options.preset === "openfga"
|
|
? spec.platformDb ? platformDbSecretStatusScript(options, spec) : openFgaSecretScript(options, spec)
|
|
: options.preset === "master-server-admin-api-key"
|
|
? masterAdminApiKeySecretScript(options, spec)
|
|
: options.preset === "bootstrap-admin"
|
|
? bootstrapAdminSecretScript(options, spec, bootstrapAdminMaterial)
|
|
: options.preset === "cloud-api-db"
|
|
? spec.platformDb ? platformDbSecretStatusScript(options, spec) : cloudApiDbSecretScript(options, spec)
|
|
: options.preset === "owned-postgres-cleanup"
|
|
? ownedPostgresCleanupScript(options, spec)
|
|
: codeAgentProviderSecretScript(options, spec, codeAgentProviderMaterial);
|
|
const result = runTransScript(options.node, script, input, options.timeoutSeconds);
|
|
const status = secretStatusFromText(statusText(result), result.exitCode === 0, result.exitCode, result.stderr, spec);
|
|
const dryRunOk = options.action === "ensure" && options.dryRun && result.exitCode === 0;
|
|
const cleanupDryRunOk = options.action === "cleanup-owned-postgres" && options.dryRun && result.exitCode === 0;
|
|
const obsoleteCleanupDryRunOk = options.action === "cleanup-obsolete" && options.dryRun && status.ok === true;
|
|
const ok = dryRunOk || cleanupDryRunOk || obsoleteCleanupDryRunOk ? true : status.ok === true;
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes secret ${options.action}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
namespace: spec.namespace,
|
|
secret: options.name,
|
|
key: options.key ?? null,
|
|
preset: options.preset,
|
|
mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "cleanup-owned-postgres" || options.action === "cleanup-obsolete" ? "confirmed-delete" : "confirmed-ensure",
|
|
status,
|
|
mutation: status.mutation === true,
|
|
result: compactCommandResult(result),
|
|
valuesRedacted: true,
|
|
next: ok && options.action === "status" ? undefined : nextSecretCommand(options, spec),
|
|
};
|
|
}
|
|
|
|
export function readCodeAgentProviderSecretMaterial(spec: RuntimeSecretSpec): CodeAgentProviderSecretMaterial {
|
|
const sourceRef = spec.codeAgentProviderSourceRef ?? null;
|
|
if (sourceRef === null) {
|
|
return {
|
|
ok: false,
|
|
sourceRef,
|
|
sourcePath: null,
|
|
sourcePresent: false,
|
|
openaiSourceKey: spec.codeAgentProviderOpenaiSourceKey ?? null,
|
|
opencodeSourceKey: spec.codeAgentProviderOpencodeSourceKey ?? null,
|
|
openaiValue: null,
|
|
opencodeValue: null,
|
|
openaiFingerprint: null,
|
|
opencodeFingerprint: null,
|
|
error: "code-agent-provider-source-ref-missing",
|
|
};
|
|
}
|
|
const paths = localSecretSourcePaths(sourceRef);
|
|
const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? null;
|
|
const openaiSourceKey = spec.codeAgentProviderOpenaiSourceKey ?? null;
|
|
const opencodeSourceKey = spec.codeAgentProviderOpencodeSourceKey ?? null;
|
|
if (sourcePath === null) {
|
|
return {
|
|
ok: false,
|
|
sourceRef,
|
|
sourcePath,
|
|
sourcePresent: false,
|
|
openaiSourceKey,
|
|
opencodeSourceKey,
|
|
openaiValue: null,
|
|
opencodeValue: null,
|
|
openaiFingerprint: null,
|
|
opencodeFingerprint: null,
|
|
error: "code-agent-provider-source-path-unresolved",
|
|
};
|
|
}
|
|
if (!existsSync(sourcePath)) {
|
|
return {
|
|
ok: false,
|
|
sourceRef,
|
|
sourcePath: displayRepoPath(sourcePath),
|
|
sourcePresent: false,
|
|
openaiSourceKey,
|
|
opencodeSourceKey,
|
|
openaiValue: null,
|
|
opencodeValue: null,
|
|
openaiFingerprint: null,
|
|
opencodeFingerprint: null,
|
|
error: "code-agent-provider-source-missing",
|
|
};
|
|
}
|
|
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
|
|
const openaiValue = openaiSourceKey === null ? null : values[openaiSourceKey] ?? null;
|
|
const opencodeValue = opencodeSourceKey === null ? null : values[opencodeSourceKey] ?? null;
|
|
const ok = (openaiValue !== null && openaiValue.length > 0) || (opencodeValue !== null && opencodeValue.length > 0);
|
|
return {
|
|
ok,
|
|
sourceRef,
|
|
sourcePath: displayRepoPath(sourcePath),
|
|
sourcePresent: true,
|
|
openaiSourceKey,
|
|
opencodeSourceKey,
|
|
openaiValue,
|
|
opencodeValue,
|
|
openaiFingerprint: openaiValue === null || openaiValue.length === 0 ? null : shortSecretFingerprint(openaiValue),
|
|
opencodeFingerprint: opencodeValue === null || opencodeValue.length === 0 ? null : shortSecretFingerprint(opencodeValue),
|
|
error: ok ? null : "code-agent-provider-source-key-missing",
|
|
};
|
|
}
|
|
|
|
export function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, string> {
|
|
if (options.action === "cleanup-owned-postgres") {
|
|
return { ensure: `bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node ${options.node} --lane ${options.lane} --confirm` };
|
|
}
|
|
if (options.action === "cleanup-obsolete") {
|
|
return { cleanup: `bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node ${options.node} --lane ${options.lane} --name ${options.name} --confirm` };
|
|
}
|
|
if (spec.externalPostgres !== undefined && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
|
|
return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` };
|
|
}
|
|
if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
|
|
return {
|
|
status: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""}`,
|
|
controlPlaneStatus: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`,
|
|
};
|
|
}
|
|
return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` };
|
|
}
|
|
|
|
export function runExternalPostgresSecretEnsure(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, unknown> {
|
|
const runtimeLaneSpec = spec.runtimeLaneSpec;
|
|
if (runtimeLaneSpec === undefined) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes secret ${options.action}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
namespace: spec.namespace,
|
|
secret: options.name,
|
|
key: options.key ?? null,
|
|
preset: options.preset,
|
|
mode: options.dryRun ? "dry-run" : "confirmed-ensure",
|
|
mutation: false,
|
|
degradedReason: "external-postgres-runtime-lane-spec-missing",
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
const sync = syncNodeExternalPostgresSecrets(runtimeLaneSpec, options.dryRun, options.timeoutSeconds);
|
|
const shouldReadStatus = sync !== null && sync.ok === true;
|
|
const statusResult = shouldReadStatus ? runTransScript(options.node, platformDbSecretStatusScript({ ...options, action: "status", dryRun: true }, spec), "", options.timeoutSeconds) : null;
|
|
const status = statusResult === null
|
|
? null
|
|
: secretStatusFromText(statusText(statusResult), statusResult.exitCode === 0, statusResult.exitCode, statusResult.stderr, spec);
|
|
const ok = sync !== null && sync.ok === true && (options.dryRun || status?.ok === true);
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes secret ${options.action}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
namespace: spec.namespace,
|
|
secret: options.name,
|
|
key: options.key ?? null,
|
|
preset: options.preset,
|
|
mode: options.dryRun ? "dry-run" : "confirmed-ensure",
|
|
status: status ?? sync,
|
|
externalPostgresSecretSync: sync,
|
|
mutation: sync?.mutation === true,
|
|
result: statusResult === null ? null : compactCommandResult(statusResult),
|
|
valuesRedacted: true,
|
|
next: ok ? undefined : nextSecretCommand(options, spec),
|
|
};
|
|
}
|
|
|
|
export function publicExposureSummary(exposure: HwlabRuntimePublicExposureSpec): Record<string, unknown> {
|
|
return {
|
|
mode: exposure.mode,
|
|
publicBaseUrl: exposure.publicBaseUrl,
|
|
hostname: exposure.hostname,
|
|
expectedA: exposure.expectedA,
|
|
frpc: {
|
|
serverAddr: exposure.serverAddr,
|
|
serverPort: exposure.serverPort,
|
|
tokenSourceRef: exposure.tokenSourceRef,
|
|
tokenSourceKey: exposure.tokenSourceKey,
|
|
secretName: exposure.secretName,
|
|
secretKey: exposure.secretKey,
|
|
tokenKey: exposure.tokenKey,
|
|
webProxy: exposure.webProxy,
|
|
apiProxy: exposure.apiProxy,
|
|
extraProxies: exposure.extraProxies,
|
|
},
|
|
caddy: {
|
|
route: exposure.caddyRoute,
|
|
configPath: exposure.caddyConfigPath,
|
|
serviceName: exposure.caddyServiceName,
|
|
tls: exposure.caddyTls,
|
|
responseHeaderTimeoutSeconds: exposure.responseHeaderTimeoutSeconds,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function runNodePublicExposure(options: NodePublicExposureOptions): Record<string, unknown> {
|
|
const exposure = options.spec.publicExposure;
|
|
if (exposure === null || !exposure.enabled) {
|
|
return {
|
|
ok: false,
|
|
command: "hwlab nodes control-plane public-exposure",
|
|
node: options.node,
|
|
lane: options.lane,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
error: "publicExposure is not configured for this node/lane target",
|
|
mutation: false,
|
|
};
|
|
}
|
|
const source = readPublicExposureTokenSource(exposure);
|
|
if (!source.ok) {
|
|
return {
|
|
ok: false,
|
|
command: "hwlab nodes control-plane public-exposure",
|
|
node: options.node,
|
|
lane: options.lane,
|
|
mode: options.dryRun ? "dry-run" : "confirmed",
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
publicExposure: publicExposureSummary(exposure),
|
|
source,
|
|
mutation: false,
|
|
valuesRedacted: true,
|
|
next: { fixSecretSource: `create .state/secrets/${exposure.tokenSourceRef} with ${exposure.tokenSourceKey}=<redacted>` },
|
|
};
|
|
}
|
|
const caddyResult = runTransHostScript(exposure.caddyRoute, publicExposureCaddyScript(options, exposure), "", options.timeoutSeconds);
|
|
const caddyFields = keyValueLinesFromText(statusText(caddyResult));
|
|
const secretStatus = options.dryRun
|
|
? publicExposureSecretDryRunStatus(options, exposure, source.value?.length ?? 0)
|
|
: publicExposureSecretApplyStatus(options, exposure, source.value ?? "");
|
|
const caddyStatus = publicExposureCaddyStatus(caddyFields, caddyResult);
|
|
const ok = secretStatus.ok === true && caddyStatus.ok === true;
|
|
return {
|
|
ok,
|
|
command: "hwlab nodes control-plane public-exposure",
|
|
node: options.node,
|
|
lane: options.lane,
|
|
mode: options.dryRun ? "dry-run" : "confirmed",
|
|
mutation: !options.dryRun && (secretStatus.mutation === true || caddyFields.mutation === "true"),
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
publicExposure: publicExposureSummary(exposure),
|
|
source: {
|
|
ok: source.ok,
|
|
path: source.path,
|
|
key: exposure.tokenSourceKey,
|
|
bytes: source.value?.length ?? 0,
|
|
fingerprint: source.fingerprint,
|
|
valueRedacted: true,
|
|
},
|
|
secret: secretStatus,
|
|
caddy: caddyStatus,
|
|
valuesRedacted: true,
|
|
next: ok
|
|
? {
|
|
triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${options.node} --lane ${options.lane} --confirm`,
|
|
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`,
|
|
}
|
|
: { retry: `bun scripts/cli.ts hwlab nodes control-plane public-exposure --node ${options.node} --lane ${options.lane} --confirm` },
|
|
};
|
|
}
|
|
|
|
function publicExposureSecretDryRunStatus(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec, tokenBytes: number): Record<string, unknown> {
|
|
return {
|
|
ok: true,
|
|
dryRun: true,
|
|
mutation: false,
|
|
planned: true,
|
|
namespace: options.spec.runtimeNamespace,
|
|
secret: exposure.secretName,
|
|
deployment: `${options.spec.runtimeNamespace}-frpc`,
|
|
beforeExists: null,
|
|
afterExists: null,
|
|
tokenBytes,
|
|
applyExitCode: null,
|
|
restartMode: null,
|
|
strategyPatchExitCode: null,
|
|
rolloutRestartExitCode: null,
|
|
scaleDownExitCode: null,
|
|
scaleDownWaitExitCode: null,
|
|
scaleDownWaitPodCount: null,
|
|
scaleUpExitCode: null,
|
|
rolloutStatusExitCode: null,
|
|
readyReplicas: null,
|
|
availableReplicas: null,
|
|
desiredReplicas: null,
|
|
strategyPatchErrorPreview: null,
|
|
restartErrorPreview: null,
|
|
scaleDownErrorPreview: null,
|
|
scaleUpErrorPreview: null,
|
|
readyErrorPreview: null,
|
|
exitCode: 0,
|
|
stderr: "",
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function publicExposureSecretApplyStatus(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec, token: string): Record<string, unknown> {
|
|
const secretApplyResult = runTransScript(options.node, publicExposureSecretScript(options, exposure), token, options.timeoutSeconds);
|
|
let secretFields = keyValueLinesFromText(statusText(secretApplyResult));
|
|
let secretResult = secretApplyResult;
|
|
if (secretApplyResult.exitCode === 0 && secretFields.afterSecretExists === "yes") {
|
|
const restartResult = runPublicExposureFrpcRecreate(options);
|
|
secretFields = { ...secretFields, ...keyValueLinesFromText(statusText(restartResult)) };
|
|
secretResult = combinePublicExposureCommandResults(secretApplyResult, restartResult);
|
|
}
|
|
return publicExposureSecretStatus(secretFields, secretResult);
|
|
}
|
|
|
|
export function readPublicExposureTokenSource(exposure: HwlabRuntimePublicExposureSpec): { ok: boolean; path: string; checkedPaths: string[]; key: string; value: string | null; fingerprint: string | null; error?: string } {
|
|
const checkedPaths = publicExposureTokenSourcePaths(exposure);
|
|
const path = checkedPaths.find((candidate) => existsSync(candidate)) ?? checkedPaths[0] ?? join(repoRoot, ".state", "secrets", exposure.tokenSourceRef);
|
|
if (!existsSync(path)) return { ok: false, path, checkedPaths, key: exposure.tokenSourceKey, value: null, fingerprint: null, error: "secret-source-missing" };
|
|
const values = parseEnvFile(readFileSync(path, "utf8"));
|
|
const value = values[exposure.tokenSourceKey];
|
|
if (value === undefined || value.length === 0) return { ok: false, path, checkedPaths, key: exposure.tokenSourceKey, value: null, fingerprint: null, error: "secret-key-missing" };
|
|
return {
|
|
ok: true,
|
|
path,
|
|
checkedPaths,
|
|
key: exposure.tokenSourceKey,
|
|
value,
|
|
fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`,
|
|
};
|
|
}
|
|
|
|
export function publicExposureTokenSourcePaths(exposure: HwlabRuntimePublicExposureSpec): string[] {
|
|
const paths = [join(repoRoot, ".state", "secrets", exposure.tokenSourceRef)];
|
|
const marker = "/.worktree/";
|
|
const index = repoRoot.indexOf(marker);
|
|
if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", exposure.tokenSourceRef));
|
|
return [...new Set(paths)];
|
|
}
|
|
|
|
export function publicExposureSecretStatus(fields: Record<string, string>, result: CommandResult): Record<string, unknown> {
|
|
const dryRun = fields.dryRun === "true";
|
|
return {
|
|
ok: result.exitCode === 0 && (dryRun || (
|
|
fields.afterSecretExists === "yes"
|
|
&& fields.strategyPatchExitCode === "0"
|
|
&& fields.rolloutRestartExitCode === "0"
|
|
&& fields.rolloutStatusExitCode === "0"
|
|
)),
|
|
dryRun,
|
|
mutation: fields.mutation === "true",
|
|
namespace: fields.namespace || null,
|
|
secret: fields.secret || null,
|
|
deployment: fields.deployment || null,
|
|
beforeExists: fields.beforeSecretExists === "yes",
|
|
afterExists: fields.afterSecretExists === "yes",
|
|
tokenBytes: numericField(fields.tokenBytes),
|
|
applyExitCode: numericField(fields.applyExitCode),
|
|
restartMode: fields.restartMode || null,
|
|
strategyPatchExitCode: numericField(fields.strategyPatchExitCode),
|
|
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
|
|
scaleDownExitCode: numericField(fields.scaleDownExitCode),
|
|
scaleDownWaitExitCode: numericField(fields.scaleDownWaitExitCode),
|
|
scaleDownWaitPodCount: numericField(fields.scaleDownWaitPodCount),
|
|
scaleUpExitCode: numericField(fields.scaleUpExitCode),
|
|
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
|
|
readyReplicas: numericField(fields.readyReplicas),
|
|
availableReplicas: numericField(fields.availableReplicas),
|
|
desiredReplicas: numericField(fields.desiredReplicas),
|
|
strategyPatchErrorPreview: fields.strategyPatchErrorPreview || null,
|
|
restartErrorPreview: fields.restartErrorPreview || null,
|
|
scaleDownErrorPreview: fields.scaleDownErrorPreview || null,
|
|
scaleUpErrorPreview: fields.scaleUpErrorPreview || null,
|
|
readyErrorPreview: fields.readyErrorPreview || null,
|
|
exitCode: result.exitCode,
|
|
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
|
|
};
|
|
}
|
|
|
|
export function publicExposureCaddyStatus(fields: Record<string, string>, result: CommandResult): Record<string, unknown> {
|
|
const dryRun = fields.dryRun === "true";
|
|
return {
|
|
ok: result.exitCode === 0
|
|
&& fields.afterBlockPresent === "yes"
|
|
&& fields.validateExitCode === "0"
|
|
&& (dryRun || (
|
|
fields.active === "active"
|
|
&& fields.localWebStatus === "200"
|
|
&& fields.localApiStatus === "200"
|
|
)),
|
|
dryRun,
|
|
mutation: fields.mutation === "true",
|
|
hostname: fields.hostname || null,
|
|
configPath: fields.configPath || null,
|
|
beforeBlockPresent: fields.beforeBlockPresent === "yes",
|
|
afterBlockPresent: fields.afterBlockPresent === "yes",
|
|
staleBlocksRemoved: numericField(fields.staleBlocksRemoved),
|
|
pythonExitCode: numericField(fields.pythonExitCode),
|
|
validateMode: fields.validateMode || null,
|
|
validateExitCode: numericField(fields.validateExitCode),
|
|
reloadExitCode: numericField(fields.reloadExitCode),
|
|
active: fields.active || null,
|
|
localWebStatus: fields.localWebStatus || null,
|
|
localApiStatus: fields.localApiStatus || null,
|
|
validateErrorPreview: fields.validateErrorPreview || null,
|
|
localWebErrorPreview: fields.localWebErrorPreview || null,
|
|
localApiErrorPreview: fields.localApiErrorPreview || null,
|
|
exitCode: result.exitCode,
|
|
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
|
|
};
|
|
}
|
|
|
|
export function publicExposureSecretScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string {
|
|
const deployment = `${options.spec.runtimeNamespace}-frpc`;
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(options.spec.runtimeNamespace)}`,
|
|
`secret=${shellQuote(exposure.secretName)}`,
|
|
`deployment=${shellQuote(deployment)}`,
|
|
`token_key=${shellQuote(exposure.tokenKey)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
"token=$(cat)",
|
|
"before_secret_exists=no",
|
|
"kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && before_secret_exists=yes",
|
|
"token_bytes=$(printf '%s' \"$token\" | wc -c | tr -d ' ')",
|
|
"apply_exit=",
|
|
"restart_mode=recreate-rollout-restart",
|
|
"strategy_patch_exit=",
|
|
"rollout_restart_exit=",
|
|
"scale_down_exit=",
|
|
"scale_down_wait_exit=",
|
|
"scale_down_wait_pod_count=",
|
|
"scale_up_exit=",
|
|
"rollout_status_exit=",
|
|
"ready_replicas=",
|
|
"available_replicas=",
|
|
"desired_replicas=",
|
|
"mutation=false",
|
|
"if [ \"$dry_run\" = false ]; then",
|
|
" tmp=$(mktemp /tmp/hwlab-public-frpc-secret.XXXXXX.yaml)",
|
|
" token_b64=$(printf '%s' \"$token\" | base64 | tr -d '\\n')",
|
|
" cat >\"$tmp\" <<EOF",
|
|
"apiVersion: v1",
|
|
"kind: Secret",
|
|
"metadata:",
|
|
" name: $secret",
|
|
" namespace: $namespace",
|
|
" labels:",
|
|
" app.kubernetes.io/part-of: hwlab",
|
|
" app.kubernetes.io/managed-by: unidesk",
|
|
"type: Opaque",
|
|
"data:",
|
|
" $token_key: $token_b64",
|
|
"EOF",
|
|
" kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-public-exposure -f \"$tmp\" >/tmp/hwlab-public-frpc-secret.apply.out 2>/tmp/hwlab-public-frpc-secret.apply.err",
|
|
" apply_exit=$?",
|
|
" rm -f \"$tmp\"",
|
|
" if [ \"$apply_exit\" = 0 ]; then mutation=true; fi",
|
|
"fi",
|
|
"after_secret_exists=no",
|
|
"kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && after_secret_exists=yes",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$secret\"",
|
|
"printf 'deployment\\t%s\\n' \"$deployment\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
|
|
"printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
|
|
"printf 'tokenBytes\\t%s\\n' \"$token_bytes\"",
|
|
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
|
"printf 'restartMode\\t%s\\n' \"$restart_mode\"",
|
|
"printf 'strategyPatchExitCode\\t%s\\n' \"$strategy_patch_exit\"",
|
|
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
|
|
"printf 'scaleDownExitCode\\t%s\\n' \"$scale_down_exit\"",
|
|
"printf 'scaleDownWaitExitCode\\t%s\\n' \"$scale_down_wait_exit\"",
|
|
"printf 'scaleDownWaitPodCount\\t%s\\n' \"$scale_down_wait_pod_count\"",
|
|
"printf 'scaleUpExitCode\\t%s\\n' \"$scale_up_exit\"",
|
|
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
|
|
"printf 'readyReplicas\\t%s\\n' \"$ready_replicas\"",
|
|
"printf 'availableReplicas\\t%s\\n' \"$available_replicas\"",
|
|
"printf 'desiredReplicas\\t%s\\n' \"$desired_replicas\"",
|
|
"if [ \"$dry_run\" = true ]; then exit 0; fi",
|
|
"[ \"$after_secret_exists\" = yes ] && [ \"$apply_exit\" = 0 ]",
|
|
].join("\n");
|
|
}
|
|
|
|
export function runPublicExposureFrpcRecreate(options: NodePublicExposureOptions): CommandResult {
|
|
const namespace = options.spec.runtimeNamespace;
|
|
const deployment = `${namespace}-frpc`;
|
|
const fields: Record<string, string> = {
|
|
deployment,
|
|
restartMode: "recreate-rollout-restart",
|
|
strategyPatchExitCode: "",
|
|
rolloutRestartExitCode: "",
|
|
scaleDownExitCode: "",
|
|
scaleDownWaitExitCode: "",
|
|
scaleDownWaitPodCount: "",
|
|
scaleUpExitCode: "",
|
|
rolloutStatusExitCode: "",
|
|
readyReplicas: "",
|
|
availableReplicas: "",
|
|
desiredReplicas: "",
|
|
strategyPatchErrorPreview: "",
|
|
restartErrorPreview: "",
|
|
scaleDownErrorPreview: "",
|
|
scaleUpErrorPreview: "",
|
|
readyErrorPreview: "",
|
|
};
|
|
const stdoutParts: string[] = [];
|
|
const stderrParts: string[] = [];
|
|
const addResult = (result: CommandResult): void => {
|
|
if (result.stdout.trim()) stdoutParts.push(result.stdout.trim());
|
|
if (result.stderr.trim()) stderrParts.push(result.stderr.trim());
|
|
};
|
|
const shortTimeout = Math.min(Math.max(20, options.timeoutSeconds), 55);
|
|
const strategyPatch = runTransScript(options.node, [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`deployment=${shellQuote(deployment)}`,
|
|
"kubectl -n \"$namespace\" patch deploy/\"$deployment\" --type=merge -p '{\"spec\":{\"strategy\":{\"type\":\"Recreate\",\"rollingUpdate\":null}}}' >/tmp/hwlab-public-frpc-strategy-patch.out 2>/tmp/hwlab-public-frpc-strategy-patch.err",
|
|
"code=$?",
|
|
"printf 'strategyPatchExitCode\\t%s\\n' \"$code\"",
|
|
"printf 'strategyPatchErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-strategy-patch.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-strategy-patch.err | cut -c1-1000 || true)\"",
|
|
"exit \"$code\"",
|
|
].join("\n"), "", shortTimeout);
|
|
addResult(strategyPatch);
|
|
Object.assign(fields, keyValueLinesFromText(statusText(strategyPatch)));
|
|
|
|
if (strategyPatch.exitCode === 0) {
|
|
const restart = runTransScript(options.node, [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`deployment=${shellQuote(deployment)}`,
|
|
"kubectl -n \"$namespace\" rollout restart deploy/\"$deployment\" >/tmp/hwlab-public-frpc-restart.out 2>/tmp/hwlab-public-frpc-restart.err",
|
|
"code=$?",
|
|
"printf 'rolloutRestartExitCode\\t%s\\n' \"$code\"",
|
|
"printf 'restartErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-restart.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-restart.err | cut -c1-1000 || true)\"",
|
|
"exit \"$code\"",
|
|
].join("\n"), "", shortTimeout);
|
|
addResult(restart);
|
|
Object.assign(fields, keyValueLinesFromText(statusText(restart)));
|
|
}
|
|
|
|
if (fields.rolloutRestartExitCode === "0") {
|
|
const deadline = Date.now() + 55_000;
|
|
while (Date.now() <= deadline) {
|
|
const probe = runTransScript(options.node, [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`deployment=${shellQuote(deployment)}`,
|
|
"kubectl -n \"$namespace\" get deploy \"$deployment\" -o jsonpath='{.spec.replicas}{\"\\t\"}{.status.readyReplicas}{\"\\t\"}{.status.availableReplicas}' >/tmp/hwlab-public-frpc-ready.out 2>/tmp/hwlab-public-frpc-ready.err",
|
|
"code=$?",
|
|
"values=$(cat /tmp/hwlab-public-frpc-ready.out 2>/dev/null || true)",
|
|
"desired=$(printf '%s' \"$values\" | cut -f1)",
|
|
"ready=$(printf '%s' \"$values\" | cut -f2)",
|
|
"available=$(printf '%s' \"$values\" | cut -f3)",
|
|
"printf 'desiredReplicas\\t%s\\n' \"$desired\"",
|
|
"printf 'readyReplicas\\t%s\\n' \"$ready\"",
|
|
"printf 'availableReplicas\\t%s\\n' \"$available\"",
|
|
"printf 'readyErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-ready.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-ready.err | cut -c1-1000 || true)\"",
|
|
"[ \"$code\" = 0 ] && [ \"$desired\" = 1 ] && [ \"$ready\" = 1 ] && [ \"$available\" = 1 ]",
|
|
].join("\n"), "", shortTimeout);
|
|
addResult(probe);
|
|
Object.assign(fields, keyValueLinesFromText(statusText(probe)));
|
|
if (probe.exitCode === 0) {
|
|
fields.rolloutStatusExitCode = "0";
|
|
break;
|
|
}
|
|
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
|
}
|
|
}
|
|
if (fields.rolloutStatusExitCode === "") fields.rolloutStatusExitCode = "1";
|
|
|
|
const ok = fields.strategyPatchExitCode === "0"
|
|
&& fields.rolloutRestartExitCode === "0"
|
|
&& fields.rolloutStatusExitCode === "0";
|
|
const stdout = Object.entries(fields).map(([key, value]) => `${key}\t${value}`).join("\n") + "\n";
|
|
return {
|
|
command: [transPath(), `${options.node}:k3s`, "sh", "--", "<public-exposure-frpc-recreate>"],
|
|
cwd: repoRoot,
|
|
exitCode: ok ? 0 : 1,
|
|
stdout: [stdout.trim(), ...stdoutParts].filter(Boolean).join("\n") + "\n",
|
|
stderr: stderrParts.join("\n"),
|
|
signal: null,
|
|
timedOut: false,
|
|
};
|
|
}
|
|
|
|
export function combinePublicExposureCommandResults(first: CommandResult, second: CommandResult): CommandResult {
|
|
return {
|
|
command: [...first.command, "&&", ...second.command],
|
|
cwd: repoRoot,
|
|
exitCode: first.exitCode === 0 && second.exitCode === 0 ? 0 : second.exitCode ?? first.exitCode,
|
|
stdout: [first.stdout.trim(), second.stdout.trim()].filter(Boolean).join("\n") + "\n",
|
|
stderr: [first.stderr.trim(), second.stderr.trim()].filter(Boolean).join("\n"),
|
|
signal: second.signal ?? first.signal,
|
|
timedOut: first.timedOut || second.timedOut,
|
|
};
|
|
}
|
|
|
|
export function publicExposureCaddyScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string {
|
|
const blockB64 = Buffer.from(publicExposureCaddyBlock(exposure), "utf8").toString("base64");
|
|
const marker = `unidesk managed ${exposure.hostname}`;
|
|
return [
|
|
"set +e",
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
`hostname=${shellQuote(exposure.hostname)}`,
|
|
`config_path=${shellQuote(exposure.caddyConfigPath)}`,
|
|
`service=${shellQuote(exposure.caddyServiceName)}`,
|
|
`marker=${shellQuote(marker)}`,
|
|
`block_b64=${shellQuote(blockB64)}`,
|
|
`expected_a=${shellQuote(exposure.expectedA)}`,
|
|
`web_port=${shellQuote(String(exposure.webProxy.remotePort))}`,
|
|
`api_port=${shellQuote(String(exposure.apiProxy.remotePort))}`,
|
|
"rm -f /tmp/hwlab-public-caddy-validate.out /tmp/hwlab-public-caddy-validate.err /tmp/hwlab-public-caddy-python.err /tmp/hwlab-public-caddy-web.err /tmp/hwlab-public-caddy-api.err",
|
|
"tmp=$(mktemp -d)",
|
|
"block=\"$tmp/block\"",
|
|
"next=\"$tmp/Caddyfile\"",
|
|
"printf '%s' \"$block_b64\" | base64 -d >\"$block\"",
|
|
"before_present=no",
|
|
"if [ -f \"$config_path\" ] && grep -Fq \"# BEGIN $marker\" \"$config_path\"; then before_present=yes; fi",
|
|
"if [ -f \"$config_path\" ]; then cp \"$config_path\" \"$next\"; else : >\"$next\"; fi",
|
|
"stale_blocks_removed=$(python3 - \"$next\" \"$block\" \"$marker\" \"$web_port\" \"$api_port\" 2>/tmp/hwlab-public-caddy-python.err <<'PY'",
|
|
"import pathlib, re, sys",
|
|
"config = pathlib.Path(sys.argv[1])",
|
|
"block = pathlib.Path(sys.argv[2]).read_text(encoding='utf-8')",
|
|
"marker = sys.argv[3]",
|
|
"web_port = sys.argv[4]",
|
|
"api_port = sys.argv[5]",
|
|
"current_name = marker[len('unidesk managed '):] if marker.startswith('unidesk managed ') else marker",
|
|
"text = config.read_text(encoding='utf-8') if config.exists() else ''",
|
|
"begin = f'# BEGIN {marker}'",
|
|
"end = f'# END {marker}'",
|
|
"managed = f'{begin}\\n{block.rstrip()}\\n{end}\\n'",
|
|
"stale_removed = [0]",
|
|
"pattern = re.compile(r'(?ms)^# BEGIN unidesk managed (?P<name>[^\\n]+)\\n(?P<body>.*?)\\n# END unidesk managed (?P=name)\\n*')",
|
|
"def keep(match):",
|
|
" name = match.group('name')",
|
|
" body = match.group('body')",
|
|
" if name != current_name and (f'127.0.0.1:{web_port}' in body or f'127.0.0.1:{api_port}' in body):",
|
|
" stale_removed[0] += 1",
|
|
" return ''",
|
|
" return match.group(0)",
|
|
"text = pattern.sub(keep, text)",
|
|
"if begin in text and end in text:",
|
|
" start = text.index(begin)",
|
|
" stop = text.index(end, start) + len(end)",
|
|
" text = text[:start].rstrip() + '\\n\\n' + managed.rstrip() + '\\n' + text[stop:].lstrip('\\n')",
|
|
"else:",
|
|
" text = text.rstrip() + '\\n\\n' + managed",
|
|
"config.write_text(text, encoding='utf-8')",
|
|
"print(stale_removed[0])",
|
|
"PY",
|
|
")",
|
|
"python_exit=$?",
|
|
"validate_exit=1",
|
|
"validate_mode=validate",
|
|
"if [ \"$python_exit\" != 0 ]; then",
|
|
" validate_mode=python",
|
|
" validate_exit=$python_exit",
|
|
"else",
|
|
" if [ \"$dry_run\" = true ]; then",
|
|
" validate_mode=adapt",
|
|
" caddy adapt --config \"$next\" --adapter caddyfile >/tmp/hwlab-public-caddy-validate.out 2>/tmp/hwlab-public-caddy-validate.err",
|
|
" else",
|
|
" sudo caddy validate --config \"$next\" --adapter caddyfile >/tmp/hwlab-public-caddy-validate.out 2>/tmp/hwlab-public-caddy-validate.err",
|
|
" fi",
|
|
" validate_exit=$?",
|
|
"fi",
|
|
"reload_exit=",
|
|
"mutation=false",
|
|
"if [ \"$dry_run\" = false ] && [ \"$validate_exit\" = 0 ]; then",
|
|
" sudo install -m 0644 \"$next\" \"$config_path\" >/tmp/hwlab-public-caddy-install.out 2>/tmp/hwlab-public-caddy-install.err",
|
|
" install_exit=$?",
|
|
" if [ \"$install_exit\" = 0 ]; then",
|
|
" mutation=true",
|
|
" sudo systemctl reload \"$service\" >/tmp/hwlab-public-caddy-reload.out 2>/tmp/hwlab-public-caddy-reload.err || sudo systemctl restart \"$service\" >>/tmp/hwlab-public-caddy-reload.out 2>>/tmp/hwlab-public-caddy-reload.err",
|
|
" reload_exit=$?",
|
|
" else",
|
|
" reload_exit=$install_exit",
|
|
" fi",
|
|
"fi",
|
|
"active=$(systemctl is-active \"$service\" 2>/dev/null || true)",
|
|
"after_present=no",
|
|
"if [ \"$dry_run\" = true ]; then grep -Fq \"# BEGIN $marker\" \"$next\" && after_present=yes; else grep -Fq \"# BEGIN $marker\" \"$config_path\" && after_present=yes; fi",
|
|
"local_web_status=",
|
|
"local_api_status=",
|
|
"if [ \"$dry_run\" = false ]; then",
|
|
" local_web_status=$(curl -kfsS --max-time 15 --resolve \"$hostname:443:127.0.0.1\" \"https://$hostname/\" -o /dev/null -w '%{http_code}' 2>/tmp/hwlab-public-caddy-web.err || true)",
|
|
" local_api_status=$(curl -kfsS --max-time 15 --resolve \"$hostname:443:127.0.0.1\" \"https://$hostname/health/live\" -o /dev/null -w '%{http_code}' 2>/tmp/hwlab-public-caddy-api.err || true)",
|
|
"fi",
|
|
"validate_err_preview=$(cat /tmp/hwlab-public-caddy-python.err /tmp/hwlab-public-caddy-validate.err 2>/dev/null | tr '\\n' ';' | cut -c1-1000 || true)",
|
|
"local_web_err_preview=$([ -f /tmp/hwlab-public-caddy-web.err ] && tr '\\n' ';' </tmp/hwlab-public-caddy-web.err 2>/dev/null | cut -c1-1000 || true)",
|
|
"local_api_err_preview=$([ -f /tmp/hwlab-public-caddy-api.err ] && tr '\\n' ';' </tmp/hwlab-public-caddy-api.err 2>/dev/null | cut -c1-1000 || true)",
|
|
"printf 'hostname\\t%s\\n' \"$hostname\"",
|
|
"printf 'expectedA\\t%s\\n' \"$expected_a\"",
|
|
"printf 'configPath\\t%s\\n' \"$config_path\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeBlockPresent\\t%s\\n' \"$before_present\"",
|
|
"printf 'afterBlockPresent\\t%s\\n' \"$after_present\"",
|
|
"printf 'staleBlocksRemoved\\t%s\\n' \"$stale_blocks_removed\"",
|
|
"printf 'pythonExitCode\\t%s\\n' \"$python_exit\"",
|
|
"printf 'validateMode\\t%s\\n' \"$validate_mode\"",
|
|
"printf 'validateExitCode\\t%s\\n' \"$validate_exit\"",
|
|
"printf 'reloadExitCode\\t%s\\n' \"$reload_exit\"",
|
|
"printf 'active\\t%s\\n' \"$active\"",
|
|
"printf 'localWebStatus\\t%s\\n' \"$local_web_status\"",
|
|
"printf 'localApiStatus\\t%s\\n' \"$local_api_status\"",
|
|
"printf 'validateErrorPreview\\t%s\\n' \"$validate_err_preview\"",
|
|
"printf 'localWebErrorPreview\\t%s\\n' \"$local_web_err_preview\"",
|
|
"printf 'localApiErrorPreview\\t%s\\n' \"$local_api_err_preview\"",
|
|
"rm -rf \"$tmp\"",
|
|
"[ \"$validate_exit\" = 0 ] && [ \"$after_present\" = yes ]",
|
|
].join("\n");
|
|
}
|
|
|
|
export function publicExposureCaddyBlock(exposure: HwlabRuntimePublicExposureSpec): string {
|
|
const tlsLines = exposure.caddyTls === "internal" ? " tls internal\n" : "";
|
|
const mainBlock = `${exposure.hostname} {
|
|
${tlsLines} @api path /health* /v1* /json-rpc* /openapi* /docs* /swagger*
|
|
reverse_proxy @api 127.0.0.1:${exposure.apiProxy.remotePort} {
|
|
transport http {
|
|
response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s
|
|
}
|
|
}
|
|
reverse_proxy 127.0.0.1:${exposure.webProxy.remotePort} {
|
|
transport http {
|
|
response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s
|
|
}
|
|
}
|
|
}`;
|
|
const extraBlocks = exposure.extraProxies
|
|
.filter((proxy) => proxy.hostname !== undefined)
|
|
.map((proxy) => `${proxy.hostname} {
|
|
${tlsLines} reverse_proxy 127.0.0.1:${proxy.remotePort} {
|
|
transport http {
|
|
response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s
|
|
}
|
|
}
|
|
}`);
|
|
return [mainBlock, ...extraBlocks].join("\n\n");
|
|
}
|
|
|
|
export function runTransScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult {
|
|
return runCommand([transPath(), `${node}:k3s`, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
|
|
}
|
|
|
|
export function runTransHostScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult {
|
|
return runCommand([transPath(), node, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
|
|
}
|
|
|
|
export function runTransWorkspaceStdinScript(node: string, workspace: string, script: string, timeoutSeconds: number): CommandResult {
|
|
return runCommand([transPath(), `${node}:${workspace}`, "sh"], repoRoot, { input: script, timeoutMs: timeoutSeconds * 1000 });
|
|
}
|
|
|
|
export function runObsoleteSecretCleanup(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, unknown> {
|
|
const kubernetesResult = runTransScript(options.node, obsoleteSecretCleanupScript(options, spec), "", options.timeoutSeconds);
|
|
const kubernetesStatus = secretStatusFromText(statusText(kubernetesResult), kubernetesResult.exitCode === 0, kubernetesResult.exitCode, kubernetesResult.stderr, spec);
|
|
const hostOptions = { ...options, dryRun: options.dryRun || kubernetesStatus.ok !== true };
|
|
const platformDbResult = runTransHostScript(options.node, obsoletePlatformDbCleanupScript(hostOptions, spec), "", options.timeoutSeconds);
|
|
const platformDbStatus = obsoletePlatformDbStatusFromText(statusText(platformDbResult), platformDbResult.exitCode === 0, platformDbResult.exitCode, platformDbResult.stderr, spec);
|
|
const ok = kubernetesStatus.ok === true && platformDbStatus.ok === true && (options.dryRun || hostOptions.dryRun === false);
|
|
const mutation = kubernetesStatus.mutation === true || platformDbStatus.mutation === true;
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes secret ${options.action}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
namespace: spec.namespace,
|
|
secret: options.name,
|
|
key: null,
|
|
preset: options.preset,
|
|
mode: options.dryRun ? "dry-run" : "confirmed-delete",
|
|
status: {
|
|
ok,
|
|
preset: "obsolete-hwpod-db-cleanup",
|
|
dryRun: options.dryRun,
|
|
mutation,
|
|
kubernetesSecret: kubernetesStatus,
|
|
platformDatabase: platformDbStatus,
|
|
hostMutationSkipped: !options.dryRun && hostOptions.dryRun,
|
|
summary: ok
|
|
? `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, and ${spec.obsoleteHwpodDbUser} are absent or ready to remove`
|
|
: `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, or ${spec.obsoleteHwpodDbUser} still needs cleanup`,
|
|
},
|
|
mutation,
|
|
result: {
|
|
kubernetesSecret: compactCommandResult(kubernetesResult),
|
|
platformDatabase: compactCommandResult(platformDbResult),
|
|
},
|
|
valuesRedacted: true,
|
|
next: ok ? undefined : nextSecretCommand(options, spec),
|
|
};
|
|
}
|