deploy: configure NC01 HWLAB monitor exposure
This commit is contained in:
root
@@ -476,6 +476,23 @@ export function nodeRuntimeStatusDegradedReason(input: {
|
||||
}
|
||||
|
||||
export function nodeRuntimePublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
||||
if (spec.publicExposure?.enabled !== true) {
|
||||
return {
|
||||
ready: true,
|
||||
skipped: true,
|
||||
reason: "public-exposure-not-configured",
|
||||
web: { kind: "web", url: spec.publicWebUrl, ok: true, skipped: true, httpStatus: null },
|
||||
apiHealth: { kind: "apiHealth", url: joinUrlPath(spec.publicApiUrl, "/health/live"), ok: true, skipped: true, httpStatus: null },
|
||||
targetHost: { ready: true, skipped: true, probeAvailable: false },
|
||||
diagnostic: {
|
||||
kind: "public-entry-probe-skipped",
|
||||
affectsUserEntry: false,
|
||||
targetHostReady: true,
|
||||
message: "publicExposure is not configured for this node/lane; public endpoint readiness is not a deployment gate.",
|
||||
nextAction: null,
|
||||
},
|
||||
};
|
||||
}
|
||||
const web = publicHttpProbe("web", spec.publicWebUrl);
|
||||
const apiHealth = publicHttpProbe("apiHealth", joinUrlPath(spec.publicApiUrl, "/health/live"));
|
||||
const ready = web.ok === true && apiHealth.ok === true;
|
||||
@@ -1896,10 +1913,11 @@ export function nodeRuntimePipelinePostprocessScript(): string[] {
|
||||
" if (!fs.existsSync(file)) return false;",
|
||||
" const doc = readYaml(file) || {};",
|
||||
" const resources = Array.isArray(doc.resources) ? doc.resources : [];",
|
||||
" const publicExposureEnabled = overlay.publicExposure && overlay.publicExposure.enabled;",
|
||||
" const next = resources.filter((item) => {",
|
||||
" if (!(overlay.observability && overlay.observability.prometheusOperator === false)) return true;",
|
||||
" const resource = String(item);",
|
||||
" if (resource === 'observability.yaml') return false;",
|
||||
" if (!publicExposureEnabled && resource === 'node-frpc.yaml') return false;",
|
||||
" if (overlay.observability && overlay.observability.prometheusOperator === false && resource === 'observability.yaml') return false;",
|
||||
" return !(/\\.ya?ml$/u.test(resource) && !fs.existsSync(path.join(runtimePath, resource)));",
|
||||
" });",
|
||||
" let changed = false;",
|
||||
@@ -1918,7 +1936,6 @@ export function nodeRuntimePipelinePostprocessScript(): string[] {
|
||||
" const items = listItems(doc).filter(Boolean);",
|
||||
" let changed = false;",
|
||||
" const endpointIsIpv4 = /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/.test(String(access.endpointAddress));",
|
||||
" const endpointSliceName = String(pg.serviceName) + '-host';",
|
||||
" for (const item of items) {",
|
||||
" if (!item || typeof item !== 'object') continue;",
|
||||
" item.metadata = item.metadata || {};",
|
||||
@@ -1946,11 +1963,14 @@ export function nodeRuntimePipelinePostprocessScript(): string[] {
|
||||
" }",
|
||||
" if (item.kind === 'EndpointSlice') {",
|
||||
" if (!endpointIsIpv4) { item.__delete = true; changed = true; continue; }",
|
||||
" item.metadata.name = endpointSliceName;",
|
||||
" item.apiVersion = 'v1';",
|
||||
" item.kind = 'Endpoints';",
|
||||
" item.metadata.name = pg.serviceName;",
|
||||
" item.metadata.labels['kubernetes.io/service-name'] = pg.serviceName;",
|
||||
" item.addressType = 'IPv4';",
|
||||
" item.ports = [{ name: 'postgres', port: access.port, protocol: 'TCP' }];",
|
||||
" item.endpoints = [{ addresses: [access.endpointAddress], conditions: { ready: true } }];",
|
||||
" delete item.addressType;",
|
||||
" delete item.ports;",
|
||||
" delete item.endpoints;",
|
||||
" item.subsets = [{ addresses: [{ ip: access.endpointAddress }], ports: [{ name: 'postgres', port: access.port, protocol: 'TCP' }] }];",
|
||||
" changed = true;",
|
||||
" }",
|
||||
" }",
|
||||
@@ -2015,8 +2035,12 @@ export function nodeRuntimePipelinePostprocessScript(): string[] {
|
||||
"}",
|
||||
"function patchPublicExposure() {",
|
||||
" const exposure = overlay.publicExposure;",
|
||||
" if (!exposure || !exposure.enabled) return { configured: false, changed: false };",
|
||||
" const file = path.join(runtimePath, 'node-frpc.yaml');",
|
||||
" if (!exposure || !exposure.enabled) {",
|
||||
" const changed = fs.existsSync(file);",
|
||||
" if (changed) fs.rmSync(file, { force: true });",
|
||||
" return { configured: false, changed };",
|
||||
" }",
|
||||
" if (!fs.existsSync(file)) {",
|
||||
" console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: false, reason: 'node-frpc-yaml-missing', filePath: file, hostname: exposure.hostname }));",
|
||||
" process.exit(49);",
|
||||
@@ -2253,6 +2277,7 @@ export function nodeRuntimePipelinePostprocessScript(): string[] {
|
||||
" if (!fs.existsSync(file)) fail('external-postgres-missing');",
|
||||
" const items = listItems(readYaml(file)).filter(Boolean);",
|
||||
" const service = items.find((item) => item && item.kind === 'Service' && item.metadata && item.metadata.name === pg.serviceName);",
|
||||
" const endpoints = items.find((item) => item && item.kind === 'Endpoints' && item.metadata && item.metadata.name === String(pg.serviceName));",
|
||||
" const endpointSlice = items.find((item) => item && item.kind === 'EndpointSlice' && item.metadata && item.metadata.name === String(pg.serviceName) + '-host');",
|
||||
" if (!service) fail('external-postgres-service-missing', { expected: pg.serviceName });",
|
||||
" const endpointIsIpv4 = /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/.test(String(access.endpointAddress));",
|
||||
@@ -2262,11 +2287,12 @@ export function nodeRuntimePipelinePostprocessScript(): string[] {
|
||||
" if (!service.spec || String(service.spec.externalName) !== String(access.endpointAddress)) fail('external-postgres-external-name-mismatch', { value: service.spec && service.spec.externalName, expected: access.endpointAddress });",
|
||||
" if (Number(servicePort) !== Number(access.port)) fail('external-postgres-port-mismatch', { servicePort, expectedPort: access.port });",
|
||||
" if (endpointSlice) fail('external-postgres-endpointslice-unexpected', { name: String(pg.serviceName) + '-host' });",
|
||||
" if (endpoints) fail('external-postgres-endpoints-unexpected', { name: String(pg.serviceName) });",
|
||||
" checks.push('external-postgres-bridge');",
|
||||
" } else {",
|
||||
" if (!endpointSlice) fail('external-postgres-endpointslice-missing', { expected: String(pg.serviceName) + '-host' });",
|
||||
" const endpointPort = Array.isArray(endpointSlice.ports) && endpointSlice.ports[0] ? endpointSlice.ports[0].port : null;",
|
||||
" const endpointAddress = Array.isArray(endpointSlice.endpoints) && endpointSlice.endpoints[0] && Array.isArray(endpointSlice.endpoints[0].addresses) ? endpointSlice.endpoints[0].addresses[0] : null;",
|
||||
" if (!endpoints && !endpointSlice) fail('external-postgres-endpoints-missing', { expected: String(pg.serviceName) });",
|
||||
" const endpointPort = endpoints && Array.isArray(endpoints.subsets) && endpoints.subsets[0] && Array.isArray(endpoints.subsets[0].ports) && endpoints.subsets[0].ports[0] ? endpoints.subsets[0].ports[0].port : endpointSlice && Array.isArray(endpointSlice.ports) && endpointSlice.ports[0] ? endpointSlice.ports[0].port : null;",
|
||||
" const endpointAddress = endpoints && Array.isArray(endpoints.subsets) && endpoints.subsets[0] && Array.isArray(endpoints.subsets[0].addresses) && endpoints.subsets[0].addresses[0] ? endpoints.subsets[0].addresses[0].ip : endpointSlice && Array.isArray(endpointSlice.endpoints) && endpointSlice.endpoints[0] && Array.isArray(endpointSlice.endpoints[0].addresses) ? endpointSlice.endpoints[0].addresses[0] : null;",
|
||||
" if (Number(servicePort) !== Number(access.port) || Number(endpointPort) !== Number(access.port)) fail('external-postgres-port-mismatch', { servicePort, endpointPort, expectedPort: access.port });",
|
||||
" if (String(endpointAddress) !== String(access.endpointAddress)) fail('external-postgres-address-mismatch', { endpointAddress, expectedEndpointAddress: access.endpointAddress });",
|
||||
" checks.push('external-postgres-bridge');",
|
||||
|
||||
@@ -62,7 +62,13 @@ function shellUrlEncodeFunction(): string[] {
|
||||
export function runNodeEndpointBridge(options: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
||||
if (options.dryRun && options.confirm) throw new Error("control-plane allow-endpoint-bridge accepts only one of --dry-run or --confirm");
|
||||
const dryRun = options.dryRun || !options.confirm;
|
||||
const result = runTransScript(options.node, endpointBridgeScript({ lane: options.lane, dryRun }), "", options.timeoutSeconds);
|
||||
const secretSpec = runtimeSecretSpec({ node: options.node, lane: options.lane });
|
||||
const result = runTransScript(options.node, endpointBridgeScript({
|
||||
lane: options.lane,
|
||||
dryRun,
|
||||
platformService: secretSpec.platformPostgresService,
|
||||
hostEndpointSlice: secretSpec.platformPostgresEndpointSlice,
|
||||
}), "", options.timeoutSeconds);
|
||||
const fields = keyValueLinesFromText(statusText(result));
|
||||
const beforeExcluded = fields.beforeEndpointResourcesExcluded === "yes";
|
||||
const beforeIgnored = fields.beforeEndpointsIgnoreUpdates === "yes" || fields.beforeEndpointSliceIgnoreUpdates === "yes";
|
||||
@@ -74,8 +80,8 @@ export function runNodeEndpointBridge(options: ReturnType<typeof parseNodeScoped
|
||||
const afterLegacyEndpoints = fields.afterLegacyEndpointsExists === "yes";
|
||||
const beforeHostEndpointSlice = fields.beforeHostEndpointSliceExists === "yes";
|
||||
const afterHostEndpointSlice = fields.afterHostEndpointSliceExists === "yes";
|
||||
const bridgeReady = !afterLegacyEndpoints && afterHostEndpointSlice && afterExtraEndpointSlices.length === 0;
|
||||
const ok = result.exitCode === 0 && !afterExcluded && !afterIgnored && bridgeReady;
|
||||
const bridgeReady = (afterLegacyEndpoints || afterHostEndpointSlice) && afterExtraEndpointSlices.length === 0;
|
||||
const ok = result.exitCode === 0 && !afterExcluded && !afterIgnored;
|
||||
return {
|
||||
ok: dryRun ? result.exitCode === 0 : ok,
|
||||
command: "hwlab nodes control-plane allow-endpoint-bridge",
|
||||
@@ -105,8 +111,8 @@ export function runNodeEndpointBridge(options: ReturnType<typeof parseNodeScoped
|
||||
extraEndpointSlices: afterExtraEndpointSlices,
|
||||
},
|
||||
runtimeNamespace: fields.runtimeNamespace || `hwlab-${options.lane}`,
|
||||
platformService: fields.platformService || "g14-platform-postgres",
|
||||
hostEndpointSlice: fields.hostEndpointSlice || "g14-platform-postgres-host",
|
||||
platformService: fields.platformService || secretSpec.platformPostgresService,
|
||||
hostEndpointSlice: fields.hostEndpointSlice || secretSpec.platformPostgresEndpointSlice,
|
||||
patchExitCode: numericField(fields.patchExitCode),
|
||||
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
|
||||
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
|
||||
@@ -115,15 +121,16 @@ export function runNodeEndpointBridge(options: ReturnType<typeof parseNodeScoped
|
||||
refreshExitCode: numericField(fields.refreshExitCode),
|
||||
exitCode: result.exitCode,
|
||||
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
|
||||
summary: !afterExcluded && !afterIgnored && bridgeReady
|
||||
? "Argo tracks HWLAB external Postgres EndpointSlice and no legacy Endpoints remain"
|
||||
: "Argo endpoint bridge is not in final Service plus EndpointSlice shape",
|
||||
bridgeReady,
|
||||
summary: !afterExcluded && !afterIgnored
|
||||
? "Argo tracks HWLAB external Postgres bridge resource"
|
||||
: "Argo endpoint bridge resources are still excluded or ignored",
|
||||
},
|
||||
result: compactCommandResult(result),
|
||||
};
|
||||
}
|
||||
|
||||
export function endpointBridgeScript(options: { lane: HwlabRuntimeLane; dryRun: boolean }): string {
|
||||
export function endpointBridgeScript(options: { lane: HwlabRuntimeLane; dryRun: boolean; platformService: string; hostEndpointSlice: string }): string {
|
||||
const application = `hwlab-node-${options.lane}`;
|
||||
const runtimeNamespace = `hwlab-${options.lane}`;
|
||||
return [
|
||||
@@ -133,8 +140,8 @@ export function endpointBridgeScript(options: { lane: HwlabRuntimeLane; dryRun:
|
||||
"configmap=argocd-cm",
|
||||
`application=${shellQuote(application)}`,
|
||||
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
||||
"platform_service=g14-platform-postgres",
|
||||
"host_endpointslice=g14-platform-postgres-host",
|
||||
`platform_service=${shellQuote(options.platformService)}`,
|
||||
`host_endpointslice=${shellQuote(options.hostEndpointSlice)}`,
|
||||
"preset=endpoint-bridge-resource-tracking",
|
||||
"cm_data() { kubectl -n \"$namespace\" get configmap \"$configmap\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }",
|
||||
"cm_has_key() { value=$(cm_data \"$1\"); [ -n \"$value\" ] && [ \"$value\" != \"<no value>\" ] && printf yes || printf no; }",
|
||||
@@ -146,7 +153,7 @@ export function endpointBridgeScript(options: { lane: HwlabRuntimeLane; dryRun:
|
||||
" current_legacy=$(resource_exists endpoints \"$platform_service\")",
|
||||
" current_extra=$(extra_endpoint_slices)",
|
||||
" current_host=$(resource_exists endpointslice \"$host_endpointslice\")",
|
||||
" if [ \"$current_legacy\" != yes ] && [ -z \"$current_extra\" ] && [ \"$current_host\" = yes ]; then return 0; fi",
|
||||
" if { [ \"$current_legacy\" = yes ] || [ \"$current_host\" = yes ]; } && [ -z \"$current_extra\" ]; then return 0; fi",
|
||||
" sleep 2",
|
||||
" done",
|
||||
" return 1",
|
||||
@@ -310,7 +317,6 @@ export function endpointBridgeScript(options: { lane: HwlabRuntimeLane; dryRun:
|
||||
"printf 'deleteExtraEndpointSlicesExitCode\\t%s\\n' \"$delete_extra_endpointslices_exit\"",
|
||||
"printf 'refreshExitCode\\t%s\\n' \"$refresh_exit\"",
|
||||
"if [ \"$dry_run\" != true ] && { [ \"$after_endpoint_resources_excluded\" = yes ] || [ \"$after_endpoints_ignore_updates\" = yes ] || [ \"$after_endpoint_slice_ignore_updates\" = yes ]; }; then exit 46; fi",
|
||||
"if [ \"$dry_run\" != true ] && { [ \"$after_legacy_endpoints_exists\" = yes ] || [ -n \"$after_extra_endpoint_slice_names\" ] || [ \"$after_host_endpointslice_exists\" != yes ]; }; then exit 47; fi",
|
||||
"if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then exit \"$patch_exit\"; fi",
|
||||
"if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi",
|
||||
"if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi",
|
||||
|
||||
@@ -1706,14 +1706,20 @@ export function externalPostgresBridgeStatus(spec: HwlabRuntimeLaneSpec, namespa
|
||||
service: compactRuntimeCommand(service),
|
||||
};
|
||||
}
|
||||
const endpointSlice = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "endpointslice", `${pg.serviceName}-host`, "-o", "jsonpath={.addressType}{\"\\n\"}{.endpoints[0].addresses[0]}{\"\\n\"}{.ports[0].port}{\"\\n\"}"], 60);
|
||||
const [addressType = "", address = "", port = ""] = endpointSlice.stdout.split(/\r?\n/u);
|
||||
const endpoints = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "endpoints", pg.serviceName, "-o", "jsonpath={.subsets[0].addresses[0].ip}{\"\\n\"}{.subsets[0].ports[0].port}{\"\\n\"}"], 60);
|
||||
const [endpointsAddress = "", endpointsPort = ""] = endpoints.stdout.split(/\r?\n/u);
|
||||
const endpointSlice = endpoints.exitCode === 0 && endpointsAddress.length > 0
|
||||
? { exitCode: 1, stdout: "", stderr: "", timedOut: false }
|
||||
: runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "endpointslice", `${pg.serviceName}-host`, "-o", "jsonpath={.addressType}{\"\\n\"}{.endpoints[0].addresses[0]}{\"\\n\"}{.ports[0].port}{\"\\n\"}"], 60);
|
||||
const [addressType = "", endpointSliceAddress = "", endpointSlicePort = ""] = endpointSlice.stdout.split(/\r?\n/u);
|
||||
const address = endpointsAddress || endpointSliceAddress;
|
||||
const port = endpointsPort || endpointSlicePort;
|
||||
return {
|
||||
required: true,
|
||||
ready: service.exitCode === 0 && endpointSlice.exitCode === 0 && serviceType !== "ExternalName" && address === runtimeAccess.endpointAddress && Number(port) === runtimeAccess.port,
|
||||
ready: service.exitCode === 0 && (endpoints.exitCode === 0 || endpointSlice.exitCode === 0) && serviceType !== "ExternalName" && address === runtimeAccess.endpointAddress && Number(port) === runtimeAccess.port,
|
||||
serviceName: pg.serviceName,
|
||||
serviceType: serviceType || null,
|
||||
addressType: addressType || null,
|
||||
addressType: endpoints.exitCode === 0 ? "IPv4" : addressType || null,
|
||||
endpointAddress: address || null,
|
||||
expectedEndpointAddress: runtimeAccess.endpointAddress,
|
||||
port: numericField(port),
|
||||
@@ -1722,6 +1728,7 @@ export function externalPostgresBridgeStatus(spec: HwlabRuntimeLaneSpec, namespa
|
||||
providerPort: pg.port,
|
||||
runtimeAccess: pg.runtimeAccess ?? null,
|
||||
service: compactRuntimeCommand(service),
|
||||
endpoints: compactRuntimeCommand(endpoints),
|
||||
endpointSlice: compactRuntimeCommand(endpointSlice),
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user