Merge pull request #1041 from pikasTech/fix/1017-runtime-secrets
fix(web-probe): sync sentinel runtime secrets
This commit is contained in:
Lyon
@@ -11,14 +11,15 @@ sentinel:
|
||||
sourceRef: hwlab/d601-v03-bootstrap-admin.env
|
||||
sourceKey: HWLAB_BOOTSTRAP_ADMIN_PASSWORD
|
||||
- purpose: account-a
|
||||
sourceRef: hwlab/web-probe-sentinel-auth-switch-account-a.env
|
||||
sourceKey: ACCOUNT_A_JSON
|
||||
sourceRef: hwlab/d601-v03-bootstrap-admin.env
|
||||
sourceKey: HWLAB_BOOTSTRAP_ADMIN_PASSWORD
|
||||
format: web-account-json
|
||||
username: admin
|
||||
- purpose: account-b
|
||||
sourceRef: hwlab/web-probe-sentinel-auth-switch-account-b.env
|
||||
sourceKey: ACCOUNT_B_JSON
|
||||
- purpose: prompt-set
|
||||
sourceRef: hwlab/web-probe-sentinel-auth-switch.env
|
||||
sourceKey: AUTH_SWITCH_UNUSED_PROMPTS_JSON
|
||||
sourceRef: hwlab/d601-v03-preset-users.env
|
||||
sourceKey: HUIGGAO_PASSWORD
|
||||
format: web-account-json
|
||||
username: huiggao
|
||||
- purpose: frp-token
|
||||
sourceRef: platform-infra/pk01-frp.env
|
||||
sourceKey: FRP_TOKEN
|
||||
@@ -35,11 +36,6 @@ sentinel:
|
||||
targetKey: account-a.json
|
||||
- sourcePurpose: account-b
|
||||
targetKey: account-b.json
|
||||
- name: hwlab-web-probe-sentinel-auth-switch-prompt-set
|
||||
namespace: hwlab-v03
|
||||
data:
|
||||
- sourcePurpose: prompt-set
|
||||
targetKey: prompts.json
|
||||
- name: hwlab-web-probe-sentinel-auth-switch-frpc
|
||||
namespace: hwlab-v03
|
||||
data:
|
||||
|
||||
@@ -99,6 +99,7 @@ interface SentinelCicdState {
|
||||
readonly runtime: Record<string, unknown>;
|
||||
readonly cicd: Record<string, unknown>;
|
||||
readonly publicExposure: Record<string, unknown>;
|
||||
readonly secrets: Record<string, unknown>;
|
||||
readonly controlPlaneTarget: Record<string, unknown>;
|
||||
readonly controlPlaneNode: Record<string, unknown>;
|
||||
readonly sourceHead: SourceHead;
|
||||
@@ -301,6 +302,7 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string |
|
||||
runtime,
|
||||
cicd,
|
||||
publicExposure,
|
||||
secrets,
|
||||
controlPlaneTarget,
|
||||
controlPlaneNode,
|
||||
sourceHead,
|
||||
@@ -634,6 +636,7 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext
|
||||
const flush = !applyOnly && record(publish).ok === true
|
||||
? runChildCli(["hwlab", "nodes", "git-mirror", "flush", "--node", state.spec.nodeId, "--lane", state.spec.lane, "--confirm", "--wait"], remainingCicdWaitSeconds())
|
||||
: null;
|
||||
const runtimeSecretsApply = applySentinelRuntimeSecrets(state, remainingCicdWaitSeconds());
|
||||
const publicExposureApply = applySentinelPublicExposure(state, remainingCicdWaitSeconds());
|
||||
const argoApply = applySentinelArgoApplication(state, remainingCicdWaitSeconds());
|
||||
const observed = waitForSentinelObservedStatus(state, remainingCicdWaitSeconds());
|
||||
@@ -645,14 +648,17 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext
|
||||
&& (applyOnly || record(sourceMirrorSync).ok === true)
|
||||
&& (applyOnly || record(publish).ok === true)
|
||||
&& (applyOnly || record(flush).ok === true)
|
||||
&& record(runtimeSecretsApply).ok === true
|
||||
&& record(publicExposureApply).ok === true
|
||||
&& record(argoApply).ok === true
|
||||
&& observedReady;
|
||||
const elapsedMs = Date.now() - startedAt;
|
||||
const blocker = ok ? null : {
|
||||
code: record(sourceMirrorSync).ok === false ? "sentinel-source-mirror-sync-failed" : "sentinel-control-plane-not-ready",
|
||||
code: record(sourceMirrorSync).ok === false ? "sentinel-source-mirror-sync-failed" : record(runtimeSecretsApply).ok === false ? "sentinel-runtime-secret-sync-failed" : "sentinel-control-plane-not-ready",
|
||||
reason: record(sourceMirrorSync).ok === false
|
||||
? "source mirror sync did not complete; investigate git mirror/proxy before control-plane publish"
|
||||
: record(runtimeSecretsApply).ok === false
|
||||
? "one or more YAML-declared runtime Secrets were not synced from sourceRef"
|
||||
: "one or more publish, publicExposure, Argo or runtime observation checks did not pass",
|
||||
};
|
||||
const result = {
|
||||
@@ -691,6 +697,7 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext
|
||||
sourceMirrorSync,
|
||||
publish,
|
||||
flush,
|
||||
runtimeSecretsApply,
|
||||
publicExposureApply,
|
||||
argoApply,
|
||||
observed,
|
||||
@@ -2162,6 +2169,127 @@ function probeSentinelPublicDashboard(state: SentinelCicdState, timeoutSeconds:
|
||||
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
|
||||
}
|
||||
|
||||
function applySentinelRuntimeSecrets(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
||||
const sourcesByPurpose = new Map<string, Record<string, unknown>>();
|
||||
for (const source of arrayAt(state.secrets, "sources").map(record)) {
|
||||
const purpose = stringAtNullable(source, "purpose");
|
||||
if (purpose !== null) sourcesByPurpose.set(purpose, source);
|
||||
}
|
||||
const desired: Array<{ namespace: string; name: string; data: Array<{ key: string; value: string; sourcePurpose: string; sourceRef: string; sourceKey: string; fingerprint: string }> }> = [];
|
||||
const missing: Record<string, unknown>[] = [];
|
||||
const skipped: Record<string, unknown>[] = [];
|
||||
for (const runtimeSecret of arrayAt(state.secrets, "runtimeSecrets").map(record)) {
|
||||
const name = stringAt(runtimeSecret, "name");
|
||||
const namespace = stringAt(runtimeSecret, "namespace");
|
||||
const data: Array<{ key: string; value: string; sourcePurpose: string; sourceRef: string; sourceKey: string; fingerprint: string }> = [];
|
||||
for (const item of arrayAt(runtimeSecret, "data").map(record)) {
|
||||
const sourcePurpose = stringAt(item, "sourcePurpose");
|
||||
const targetKey = stringAt(item, "targetKey");
|
||||
if (sourcePurpose === "frp-token") {
|
||||
skipped.push({ name, namespace, targetKey, sourcePurpose, reason: "managed-by-publicExposure-frpc", valuesRedacted: true });
|
||||
continue;
|
||||
}
|
||||
const source = sourcesByPurpose.get(sourcePurpose);
|
||||
if (source === undefined) {
|
||||
missing.push({ name, namespace, targetKey, sourcePurpose, reason: "source-purpose-missing", valuesRedacted: true });
|
||||
continue;
|
||||
}
|
||||
const sourceRef = stringAt(source, "sourceRef");
|
||||
const sourceKey = stringAt(source, "sourceKey");
|
||||
const material = readSentinelSecretSourceValue(source);
|
||||
if (!material.ok) {
|
||||
missing.push({ name, namespace, targetKey, sourcePurpose, sourceRef, sourceKey, reason: material.error, sourcePath: material.sourcePath, valuesRedacted: true });
|
||||
continue;
|
||||
}
|
||||
const value = stringAt(material, "value");
|
||||
data.push({
|
||||
key: targetKey,
|
||||
value,
|
||||
sourcePurpose,
|
||||
sourceRef,
|
||||
sourceKey,
|
||||
fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`,
|
||||
});
|
||||
}
|
||||
if (data.length > 0) desired.push({ namespace, name, data });
|
||||
}
|
||||
if (missing.length > 0) return { ok: false, phase: "local-source", missing, skipped, valuesRedacted: true };
|
||||
if (desired.length === 0) return { ok: true, phase: "skipped-no-runtime-secrets", secretCount: 0, keyCount: 0, skippedKeyCount: skipped.length, skipped, valuesRedacted: true };
|
||||
const manifests = desired.map((secret) => ({
|
||||
apiVersion: "v1",
|
||||
kind: "Secret",
|
||||
metadata: {
|
||||
name: secret.name,
|
||||
namespace: secret.namespace,
|
||||
labels: {
|
||||
"app.kubernetes.io/managed-by": "unidesk",
|
||||
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
||||
"unidesk.ai/node": state.spec.nodeId,
|
||||
"unidesk.ai/lane": state.spec.lane,
|
||||
"unidesk.ai/web-probe-sentinel-id": state.sentinelId,
|
||||
},
|
||||
},
|
||||
type: "Opaque",
|
||||
data: Object.fromEntries(secret.data.map((item) => [item.key, Buffer.from(item.value, "utf8").toString("base64")])),
|
||||
}));
|
||||
const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
|
||||
const summary = desired.map((secret) => ({
|
||||
name: secret.name,
|
||||
namespace: secret.namespace,
|
||||
keys: secret.data.map((item) => item.key).sort(),
|
||||
sources: secret.data.map((item) => ({ key: item.key, sourcePurpose: item.sourcePurpose, sourceRef: item.sourceRef, sourceKey: item.sourceKey, fingerprint: item.fingerprint, valuesRedacted: true })),
|
||||
valuesRedacted: true,
|
||||
}));
|
||||
const summaryB64 = Buffer.from(JSON.stringify(summary), "utf8").toString("base64");
|
||||
const script = [
|
||||
"set +e",
|
||||
`summary_b64=${shellQuote(summaryB64)}`,
|
||||
"tmp=$(mktemp -d)",
|
||||
"trap 'rm -rf \"$tmp\"' EXIT",
|
||||
"manifest=\"$tmp/runtime-secrets.yaml\"",
|
||||
"cat >\"$manifest\"",
|
||||
"kubectl apply --server-side --force-conflicts --field-manager=unidesk-web-probe-sentinel-runtime-secrets -f \"$manifest\" >/tmp/web-probe-sentinel-runtime-secrets.out 2>/tmp/web-probe-sentinel-runtime-secrets.err",
|
||||
"apply_rc=$?",
|
||||
"python3 - \"$summary_b64\" \"$apply_rc\" <<'PY'",
|
||||
"import base64, json, subprocess, sys",
|
||||
"summary = json.loads(base64.b64decode(sys.argv[1]).decode('utf-8'))",
|
||||
"apply_rc = int(sys.argv[2])",
|
||||
"items = []",
|
||||
"ok = apply_rc == 0",
|
||||
"for secret in summary:",
|
||||
" proc = subprocess.run(['kubectl', '-n', secret['namespace'], 'get', 'secret', secret['name'], '-o', 'json'], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)",
|
||||
" data = json.loads(proc.stdout).get('data', {}) if proc.returncode == 0 and proc.stdout else {}",
|
||||
" keys = {key: key in data for key in secret['keys']}",
|
||||
" present = proc.returncode == 0 and all(keys.values())",
|
||||
" ok = ok and present",
|
||||
" items.append({'name': secret['name'], 'namespace': secret['namespace'], 'present': present, 'keys': keys, 'sources': secret['sources'], 'valuesRedacted': True})",
|
||||
"print(json.dumps({'ok': ok, 'applyExitCode': apply_rc, 'secretCount': len(summary), 'keyCount': sum(len(item['keys']) for item in summary), 'items': items, 'valuesRedacted': True}, ensure_ascii=False))",
|
||||
"PY",
|
||||
].join("\n");
|
||||
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { input: manifestYaml, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
||||
const parsed = parseJsonObject(result.stdout);
|
||||
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), skippedKeyCount: skipped.length, skipped, result: compactCommand(result), valuesRedacted: true };
|
||||
}
|
||||
|
||||
function readSentinelSecretSourceValue(source: Record<string, unknown>): Record<string, unknown> {
|
||||
const sourceRef = stringAt(source, "sourceRef");
|
||||
const sourceKey = stringAt(source, "sourceKey");
|
||||
const paths = secretSourcePaths(sourceRef);
|
||||
const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
|
||||
if (!existsSync(sourcePath)) return { ok: false, error: "secret-source-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
||||
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
|
||||
const value = values[sourceKey];
|
||||
if (value === undefined || value.length === 0) return { ok: false, error: "secret-source-key-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
||||
const format = stringAtNullable(source, "format");
|
||||
if (format === null) return { ok: true, value, sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
||||
if (format === "web-account-json") {
|
||||
const username = stringAtNullable(source, "username");
|
||||
if (username === null) return { ok: false, error: "web-account-json-username-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
||||
return { ok: true, value: JSON.stringify({ username, password: value }), sourceRef, sourceKey, format, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
||||
}
|
||||
return { ok: false, error: "unsupported-secret-source-format", sourceRef, sourceKey, format, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
||||
}
|
||||
|
||||
function applySentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
||||
const material = readSentinelFrpcMaterial(state);
|
||||
if (!material.ok) return { ok: false, hostname: stringAt(state.publicExposure, "hostname"), material, valuesRedacted: true };
|
||||
@@ -3126,6 +3254,7 @@ function renderControlPlaneResult(result: Record<string, unknown>): string {
|
||||
const sourceMirrorSync = record(result.sourceMirrorSync);
|
||||
const publish = record(result.publish);
|
||||
const flush = record(result.flush);
|
||||
const runtimeSecretsApply = record(result.runtimeSecretsApply);
|
||||
const publicExposureApply = record(result.publicExposureApply);
|
||||
const publicExposureCaddy = record(publicExposureApply.caddy);
|
||||
const argoApply = record(result.argoApply);
|
||||
@@ -3163,6 +3292,8 @@ function renderControlPlaneResult(result: Record<string, unknown>): string {
|
||||
"",
|
||||
Object.keys(flush).length === 0 ? "FLUSH\n-" : table(["OK", "EXIT", "TIMED_OUT", "PREVIEW"], [[flush.ok, record(flush.result).exitCode, record(flush.result).timedOut, record(flush.result).stdoutPreview]]),
|
||||
"",
|
||||
Object.keys(runtimeSecretsApply).length === 0 ? "RUNTIME_SECRETS\n-" : table(["OK", "SECRETS", "KEYS", "SKIPPED"], [[runtimeSecretsApply.ok, runtimeSecretsApply.secretCount ?? "-", runtimeSecretsApply.keyCount ?? "-", runtimeSecretsApply.skippedKeyCount ?? "-"]]),
|
||||
"",
|
||||
Object.keys(publicExposureApply).length === 0 ? "PUBLIC_EXPOSURE_APPLY\n-" : table(["OK", "SECRET", "CADDY", "HOST"], [[publicExposureApply.ok, record(publicExposureApply.secret).ok, record(publicExposureApply.caddy).ok, publicExposureApply.hostname]]),
|
||||
"",
|
||||
Object.keys(publicExposureCaddy).length === 0 || publicExposureCaddy.ok === true
|
||||
|
||||
Reference in New Issue
Block a user