feat: 接入 selfmedia 受控交付

scripts/src/secrets.ts

@@ -1,5 +1,6 @@
-import { randomBytes } from "node:crypto";
+import { createHash, randomBytes } from "node:crypto";
 import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
 import { dirname, isAbsolute, join } from "node:path";
 import type { UniDeskConfig } from "./config";
 import { rootPath } from "./config";
@@ -39,23 +40,36 @@ interface SecretDistributionConfig {
   version: number;
   kind: "unidesk-secret-distribution";
   metadata: { id: string; owner: string; relatedIssues: number[] };
-  sources: { root: string; files: SourceFileConfig[] };
+  sources: { root: string; files: EnvSourceFileConfig[]; externalFiles: RawSourceFileConfig[] };
   targets: DistributionTarget[];
   kubernetesSecrets: KubernetesSecretConfig[];
 }
 
-interface SourceFileConfig {
+interface SourceCreateIfMissingConfig {
+  enabled: boolean;
+  values: Record<string, string>;
+  randomHex: Record<string, number>;
+  randomBase64Url: Record<string, { bytes: number; prefix: string }>;
+}
+
+interface EnvSourceFileConfig {
   sourceRef: string;
   type: "env";
   requiredKeys: string[];
-  createIfMissing: {
-    enabled: boolean;
-    values: Record<string, string>;
-    randomHex: Record<string, number>;
-    randomBase64Url: Record<string, { bytes: number; prefix: string }>;
-  };
+  required: true;
+  createIfMissing: SourceCreateIfMissingConfig;
 }
 
+interface RawSourceFileConfig {
+  sourceRef: string;
+  type: "raw-file";
+  requiredKeys: ["contents"];
+  required: boolean;
+  createIfMissing: SourceCreateIfMissingConfig;
+}
+
+type SourceFileConfig = EnvSourceFileConfig | RawSourceFileConfig;
+
 interface DistributionTarget {
   id: string;
   route: string;
@@ -80,8 +94,10 @@ interface SecretDataMapping {
 
 interface SourceMaterial {
   sourceRef: string;
+  type: SourceFileConfig["type"];
   sourcePath: string;
   exists: boolean;
+  required: boolean;
   requiredKeys: string[];
   presentKeys: string[];
   missingKeys: string[];
@@ -190,7 +206,7 @@ function parseOptions(args: string[]): SecretsOptions {
 function plan(options: SecretsOptions): Record<string, unknown> {
   const distribution = readSecretDistributionConfig(options.configPath);
   assertSelectedTargets(distribution, options);
-  const sources = inspectSources(distribution, false);
+  const sources = inspectSources(distribution, false, selectedSourceRefs(distribution, options));
   const desired = desiredSecrets(distribution, options, sources);
   return {
     ok: sources.ok && desired.every((secret) => secret.missingKeys.length === 0),
@@ -200,7 +216,7 @@ function plan(options: SecretsOptions): Record<string, unknown> {
     localSources: sourceSummary(sources),
     desiredSecrets: desired.map(desiredSecretSummary),
     policy: {
-      sourceAuthority: "local YAML-declared sourceRef files under sources.root",
+      sourceAuthority: "local YAML-declared managed and external sourceRef files",
       runtimeReverseEngineering: false,
       valuesPrinted: false,
     },
@@ -233,7 +249,7 @@ async function sync(config: UniDeskConfig, options: SecretsOptions): Promise<Rec
       statusCommand: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`,
     };
   }
-  const sources = inspectSources(distribution, true);
+  const sources = inspectSources(distribution, true, selectedSourceRefs(distribution, options));
   if (!sources.ok) return { ok: false, action: "secrets-sync", mode: "blocked-local-sources", mutation: true, config: configSummary(distribution, options), localSources: sourceSummary(sources) };
   const desired = desiredSecrets(distribution, options, sources);
   const missing = desired.flatMap((secret) => secret.missingKeys);
@@ -257,7 +273,7 @@ async function sync(config: UniDeskConfig, options: SecretsOptions): Promise<Rec
 async function status(config: UniDeskConfig, options: SecretsOptions): Promise<Record<string, unknown>> {
   const distribution = readSecretDistributionConfig(options.configPath);
   assertSelectedTargets(distribution, options);
-  const sources = inspectSources(distribution, false);
+  const sources = inspectSources(distribution, false, selectedSourceRefs(distribution, options));
   const desired = desiredSecrets(distribution, options, sources);
   const perTarget = await Promise.all(groupDesiredSecretsByTarget(desired).map(async (group) => await statusTargetSecrets(config, group.target, group.secrets, options)));
   return {
@@ -293,6 +309,9 @@ function readSecretDistributionConfig(pathArg: string): SecretDistributionConfig
     sources: {
       root: stringField(sources, "root", `${label}.sources`),
       files: arrayOfRecords(sources.files, `${label}.sources.files`).map((item, index) => parseSourceFile(item, `${label}.sources.files[${index}]`)),
+      externalFiles: sources.externalFiles === undefined
+        ? []
+        : arrayOfRecords(sources.externalFiles, `${label}.sources.externalFiles`).map((item, index) => parseRawSourceFile(item, `${label}.sources.externalFiles[${index}]`)),
     },
     targets: arrayOfRecords(root.targets, `${label}.targets`).map((item, index) => parseTarget(item, `${label}.targets[${index}]`)),
     kubernetesSecrets: arrayOfRecords(root.kubernetesSecrets, `${label}.kubernetesSecrets`).map((item, index) => parseKubernetesSecret(item, `${label}.kubernetesSecrets[${index}]`)),
@@ -301,7 +320,7 @@ function readSecretDistributionConfig(pathArg: string): SecretDistributionConfig
   return config;
 }
 
-function parseSourceFile(record: Record<string, unknown>, path: string): SourceFileConfig {
+function parseSourceFile(record: Record<string, unknown>, path: string): EnvSourceFileConfig {
   const type = stringField(record, "type", path);
   if (type !== "env") throw new Error(`${path}.type must be env`);
   const createRaw = record.createIfMissing === undefined ? {} : objectField(record, "createIfMissing", path);
@@ -310,6 +329,7 @@ function parseSourceFile(record: Record<string, unknown>, path: string): SourceF
     sourceRef: sourceRefField(record, "sourceRef", path),
     type,
     requiredKeys: stringArrayField(record, "requiredKeys", path).map((key, index) => envKeyValue(key, `${path}.requiredKeys[${index}]`)),
+    required: true,
     createIfMissing: {
       enabled: createRaw.enabled === undefined ? false : booleanField(createRaw, "enabled", `${path}.createIfMissing`),
       values: createRaw.values === undefined ? {} : stringMapField(createRaw, "values", `${path}.createIfMissing`),
@@ -319,6 +339,35 @@ function parseSourceFile(record: Record<string, unknown>, path: string): SourceF
   };
 }
 
+function parseRawSourceFile(record: Record<string, unknown>, path: string): RawSourceFileConfig {
+  const type = stringField(record, "type", path);
+  if (type !== "raw-file") throw new Error(`${path}.type must be raw-file`);
+  const createRaw = objectField(record, "createIfMissing", path);
+  const enabled = booleanField(createRaw, "enabled", `${path}.createIfMissing`);
+  const randomHex = createRaw.randomHex === undefined
+    ? {}
+    : { contents: randomByteCount(createRaw.randomHex, `${path}.createIfMissing.randomHex`) };
+  const randomBase64Url = createRaw.randomBase64Url === undefined
+    ? {}
+    : { contents: randomBase64UrlSpec(createRaw.randomBase64Url, `${path}.createIfMissing.randomBase64Url`) };
+  if (createRaw.values !== undefined) throw new Error(`${path}.createIfMissing.values is not allowed for raw-file sources`);
+  const generatorCount = Object.keys(randomHex).length + Object.keys(randomBase64Url).length;
+  if (enabled && generatorCount !== 1) throw new Error(`${path}.createIfMissing must declare exactly one of randomHex or randomBase64Url when enabled`);
+  if (!enabled && generatorCount !== 0) throw new Error(`${path}.createIfMissing generators require enabled: true`);
+  return {
+    sourceRef: externalSourceRefField(record, "sourceRef", path),
+    type,
+    requiredKeys: ["contents"],
+    required: booleanField(record, "required", path),
+    createIfMissing: {
+      enabled,
+      values: {},
+      randomHex,
+      randomBase64Url,
+    },
+  };
+}
+
 function parseTarget(record: Record<string, unknown>, path: string): DistributionTarget {
   return {
     id: simpleId(stringField(record, "id", path), `${path}.id`),
@@ -338,17 +387,22 @@ function parseKubernetesSecret(record: Record<string, unknown>, path: string): K
     secretName: kubernetesNameField(record, "secretName", path),
     type,
     data: arrayOfRecords(record.data, `${path}.data`).map((item, index) => ({
-      sourceRef: sourceRefField(item, "sourceRef", `${path}.data[${index}]`),
-      sourceKey: envKeyField(item, "sourceKey", `${path}.data[${index}]`),
+      sourceRef: declaredSourceRefField(item, "sourceRef", `${path}.data[${index}]`),
+      sourceKey: sourceDataKeyField(item, "sourceKey", `${path}.data[${index}]`),
       targetKey: kubernetesSecretKeyField(item, "targetKey", `${path}.data[${index}]`),
     })),
   };
 }
 
 function validateDistributionConfig(config: SecretDistributionConfig): void {
-  if (config.sources.files.length === 0) throw new Error(`${config.configPath}.sources.files must not be empty`);
+  const sourceFiles = allSourceFiles(config);
+  if (sourceFiles.length === 0) throw new Error(`${config.configPath}.sources must declare files or externalFiles`);
   if (config.targets.length === 0) throw new Error(`${config.configPath}.targets must not be empty`);
-  const sources = new Map(config.sources.files.map((item) => [item.sourceRef, item]));
+  const sources = new Map<string, SourceFileConfig>();
+  for (const source of sourceFiles) {
+    if (sources.has(source.sourceRef)) throw new Error(`${config.configPath}.sources declares duplicate sourceRef ${source.sourceRef}`);
+    sources.set(source.sourceRef, source);
+  }
   const targets = new Set(config.targets.map((item) => item.id));
   const targetSecrets = new Set<string>();
   for (const secret of config.kubernetesSecrets) {
@@ -360,20 +414,25 @@ function validateDistributionConfig(config: SecretDistributionConfig): void {
     for (const item of secret.data) {
       const source = sources.get(item.sourceRef);
       if (source === undefined) throw new Error(`${config.configPath}.kubernetesSecrets.${secret.name} references undeclared sourceRef ${item.sourceRef}`);
-      if (!source.requiredKeys.includes(item.sourceKey)) throw new Error(`${config.configPath}.kubernetesSecrets.${secret.name} maps ${item.sourceRef}.${item.sourceKey}, but that key is not listed in sources.files.requiredKeys`);
+      if (!source.requiredKeys.includes(item.sourceKey)) throw new Error(`${config.configPath}.kubernetesSecrets.${secret.name} maps undeclared source key ${item.sourceRef}.${item.sourceKey}`);
       if (targetKeys.has(item.targetKey)) throw new Error(`${config.configPath}.kubernetesSecrets.${secret.name} maps duplicate target key ${item.targetKey}`);
       targetKeys.add(item.targetKey);
     }
   }
 }
 
-function inspectSources(config: SecretDistributionConfig, materialize: boolean): SourceInspection {
+function allSourceFiles(config: SecretDistributionConfig): SourceFileConfig[] {
+  return [...config.sources.files, ...config.sources.externalFiles];
+}
+
+function inspectSources(config: SecretDistributionConfig, materialize: boolean, selectedRefs: Set<string>): SourceInspection {
   const root = secretRoot(config);
   const materials = new Map<string, SourceMaterial>();
-  const entries = config.sources.files.map((source) => {
-    const sourcePath = join(root, source.sourceRef);
+  const entries = allSourceFiles(config).filter((source) => selectedRefs.has(source.sourceRef)).map((source) => {
+    const sourcePath = source.type === "env" ? join(root, source.sourceRef) : externalSourcePath(source.sourceRef);
     const exists = existsSync(sourcePath);
-    const existing = exists ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {};
+    const existingText = exists ? readFileSync(sourcePath, "utf8") : "";
+    const existing = exists ? source.type === "env" ? parseEnvFile(existingText) : { contents: existingText } : {};
     const next = { ...existing };
     const generatedKeys: string[] = [];
     const unmaterializedGeneratedKeys: string[] = [];
@@ -402,11 +461,30 @@ function inspectSources(config: SecretDistributionConfig, materialize: boolean):
     const missingBefore = source.requiredKeys.filter((key) => existing[key] === undefined || existing[key].length === 0);
     const missingKeys = source.requiredKeys.filter((key) => next[key] === undefined || next[key].length === 0);
     const action = missingKeys.length > 0 ? "blocked" : !exists ? "create" : missingBefore.length > 0 ? "update" : "none";
-    if (materialize && action !== "blocked" && (action === "create" || action === "update")) writeEnvFile(sourcePath, next);
+    if (materialize && action !== "blocked" && (action === "create" || action === "update")) {
+      if (source.type === "env") writeEnvFile(sourcePath, next);
+      else writeRawFile(sourcePath, next.contents);
+    }
+    const materialized = materialize && action !== "blocked" && (action === "create" || action === "update");
+    const presence = materialized || (exists && existingText.length > 0);
+    const bytes = materialized
+      ? source.type === "env"
+        ? Buffer.byteLength(readFileSync(sourcePath, "utf8"), "utf8")
+        : Buffer.byteLength(next.contents, "utf8")
+      : exists
+        ? Buffer.byteLength(existingText, "utf8")
+        : null;
+    const fingerprint = missingKeys.length === 0 && unmaterializedGeneratedKeys.length === 0
+      ? source.type === "raw-file"
+        ? fingerprintRawFileContent(next.contents)
+        : fingerprintValues(next, source.requiredKeys)
+      : null;
     const material: SourceMaterial = {
       sourceRef: source.sourceRef,
+      type: source.type,
       sourcePath,
       exists,
+      required: source.required,
       requiredKeys: source.requiredKeys,
       presentKeys: source.requiredKeys.filter((key) => next[key] !== undefined && next[key].length > 0),
       missingKeys,
@@ -414,11 +492,22 @@ function inspectSources(config: SecretDistributionConfig, materialize: boolean):
       generatedKeys,
       unmaterializedGeneratedKeys,
       values: next,
-      fingerprint: missingKeys.length === 0 && unmaterializedGeneratedKeys.length === 0 ? fingerprintValues(next, source.requiredKeys) : null,
+      fingerprint,
     };
     materials.set(source.sourceRef, material);
+    if (source.type === "raw-file") {
+      return {
+        sourceRef: source.sourceRef,
+        type: source.type,
+        presence,
+        bytes,
+        fingerprint,
+        valuesPrinted: false,
+      };
+    }
     return {
       sourceRef: source.sourceRef,
+      type: source.type,
       sourcePath: redactRepoPath(sourcePath),
       exists,
       requiredKeys: source.requiredKeys,
@@ -432,7 +521,7 @@ function inspectSources(config: SecretDistributionConfig, materialize: boolean):
     };
   });
   return {
-    ok: entries.every((entry) => (entry.missingKeys as string[]).length === 0),
+    ok: Array.from(materials.values()).every((material) => !material.required || material.missingKeys.length === 0),
     root: redactRepoPath(root),
     entries,
     materials,
@@ -489,6 +578,13 @@ function selectedTargets(config: SecretDistributionConfig, options: SecretsOptio
     .filter((target) => options.targetId === null || target.id === options.targetId);
 }
 
+function selectedSourceRefs(config: SecretDistributionConfig, options: SecretsOptions): Set<string> {
+  const targetIds = new Set(selectedTargets(config, options).map((target) => target.id));
+  return new Set(config.kubernetesSecrets
+    .filter((secret) => targetIds.has(secret.targetId))
+    .flatMap((secret) => secret.data.map((item) => item.sourceRef)));
+}
+
 async function applyTargetSecrets(config: UniDeskConfig, target: DistributionTarget, secrets: DesiredSecret[], options: SecretsOptions): Promise<Record<string, unknown>> {
   if (secrets.length === 0) return { ok: true, target: targetSummary(target), mode: "skipped-no-secrets" };
   const yaml = renderSecretManifest(target, secrets);
@@ -561,21 +657,21 @@ else
   apply_rc=1
 fi
 python3 - "$ns_rc" "$apply_rc" "$tmp/ns.out" "$tmp/ns.err" "$tmp/apply.out" "$tmp/apply.err" <<'PY'
-import base64, json, sys
+import base64, json, os, sys
 ns_rc, apply_rc = int(sys.argv[1]), int(sys.argv[2])
-def text(path, limit=5000):
+def byte_count(path):
     try:
-        return open(path, encoding="utf-8", errors="replace").read()[-limit:]
+        return os.path.getsize(path)
     except FileNotFoundError:
-        return ""
+        return 0
 payload = {
     "ok": ns_rc == 0 and apply_rc == 0,
     "namespace": "${target.namespace}",
     "secrets": json.loads(base64.b64decode("${summaryB64}").decode("utf-8")),
     "valuesPrinted": False,
     "steps": {
-        "namespace": {"exitCode": ns_rc, "stdout": text(sys.argv[3]), "stderr": text(sys.argv[4])},
-        "apply": {"exitCode": apply_rc, "stdout": text(sys.argv[5]), "stderr": text(sys.argv[6])},
+        "namespace": {"exitCode": ns_rc, "stdoutBytes": byte_count(sys.argv[3]), "stderrBytes": byte_count(sys.argv[4]), "outputOmitted": True},
+        "apply": {"exitCode": apply_rc, "stdoutBytes": byte_count(sys.argv[5]), "stderrBytes": byte_count(sys.argv[6]), "outputOmitted": True},
     },
 }
 print(json.dumps(payload, ensure_ascii=False, indent=2))
@@ -645,13 +741,17 @@ function groupDesiredSecretsByTarget(secrets: DesiredSecret[]): Array<{ target:
 }
 
 function configSummary(config: SecretDistributionConfig, options: SecretsOptions): Record<string, unknown> {
+  const sourceRefs = selectedSourceRefs(config, options);
   return {
     path: config.configPath,
     metadata: config.metadata,
     root: redactRepoPath(secretRoot(config)),
     scope: options.scope,
     targetId: options.targetId,
-    sources: config.sources.files.map((item) => ({ sourceRef: item.sourceRef, requiredKeys: item.requiredKeys, createIfMissing: item.createIfMissing.enabled })),
+    sources: [
+      ...config.sources.files.filter((item) => sourceRefs.has(item.sourceRef)).map((item) => ({ sourceRef: item.sourceRef, type: item.type, requiredKeys: item.requiredKeys, createIfMissing: item.createIfMissing.enabled })),
+      ...config.sources.externalFiles.filter((item) => sourceRefs.has(item.sourceRef)).map((item) => ({ sourceRef: item.sourceRef, type: item.type, required: item.required, createIfMissing: item.createIfMissing.enabled })),
+    ],
     targets: config.targets.filter((target) => (options.scope === null || target.scope === options.scope) && (options.targetId === null || target.id === options.targetId)).map(targetSummary),
     valuesPrinted: false,
   };
@@ -748,6 +848,16 @@ function writeEnvFile(path: string, values: Record<string, string>): void {
   chmodSync(path, 0o600);
 }
 
+function writeRawFile(path: string, value: string): void {
+  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+  writeFileSync(path, value, { encoding: "utf8", mode: 0o600 });
+  chmodSync(path, 0o600);
+}
+
+function fingerprintRawFileContent(value: string): string {
+  return `sha256:${createHash("sha256").update(value, "utf8").digest("hex")}`;
+}
+
 function quoteEnv(value: string): string {
   if (/^[A-Za-z0-9_./:@%?&=+-]+$/u.test(value)) return value;
   return `'${value.replaceAll("'", "'\"'\"'")}'`;
@@ -837,8 +947,30 @@ function sourceRefField(obj: Record<string, unknown>, key: string, path: string)
   return value;
 }
 
-function envKeyField(obj: Record<string, unknown>, key: string, path: string): string {
-  return envKeyValue(stringField(obj, key, path), `${path}.${key}`);
+function externalSourceRefField(obj: Record<string, unknown>, key: string, path: string): string {
+  const value = stringField(obj, key, path);
+  if (value.includes("\0") || value.split("/").includes("..")) throw new Error(`${path}.${key} must not contain NUL or parent traversal`);
+  if (value.startsWith("~/") && value.length > 2) return value;
+  if (isAbsolute(value)) return value;
+  throw new Error(`${path}.${key} must be an absolute path or a ~/ home path`);
+}
+
+function declaredSourceRefField(obj: Record<string, unknown>, key: string, path: string): string {
+  const value = stringField(obj, key, path);
+  if (value.startsWith("~/") || isAbsolute(value)) return externalSourceRefField(obj, key, path);
+  return sourceRefField(obj, key, path);
+}
+
+function externalSourcePath(sourceRef: string): string {
+  if (sourceRef.startsWith("~/")) return join(homedir(), sourceRef.slice(2));
+  if (isAbsolute(sourceRef)) return sourceRef;
+  throw new Error(`external sourceRef ${sourceRef} must be an absolute path or a ~/ home path`);
+}
+
+function sourceDataKeyField(obj: Record<string, unknown>, key: string, path: string): string {
+  const value = stringField(obj, key, path);
+  if (!/^[A-Za-z_][A-Za-z0-9._-]*$/u.test(value)) throw new Error(`${path}.${key} must be a source data key`);
+  return value;
 }
 
 function envKeyValue(value: string, path: string): string {