fix: 使用持久源码缓存构建 SelfMedia 语音镜像 (#2180)
* fix: 固化语音源码构建缓存 * fix: 对齐 D518 语音源码缓存路径 --------- Co-authored-by: Codex <codex@local>
This commit is contained in:
@@ -18,6 +18,7 @@ interface VoiceTarget {
|
||||
composeProject: string;
|
||||
cacheDir?: string;
|
||||
dataDir?: string;
|
||||
sourceDir?: string;
|
||||
}
|
||||
|
||||
interface VoiceConfig {
|
||||
@@ -25,11 +26,11 @@ interface VoiceConfig {
|
||||
kind: "platform-infra-selfmedia-voice";
|
||||
metadata: { id: string; owner: string; relatedIssues: number[]; mdtodo: string };
|
||||
rollout: { prerequisite: string; approved: boolean };
|
||||
artifacts: Record<"root" | "containerfile" | "api" | "modelInit" | "d518Compose" | "nc01Compose" | "frpcTemplate" | "frpsTemplate", string>;
|
||||
artifacts: Record<"root" | "containerfile" | "api" | "modelInit" | "sourceInit" | "d518Compose" | "nc01Compose" | "frpcTemplate" | "frpsTemplate", string>;
|
||||
image: {
|
||||
name: string;
|
||||
base: { repository: string; tag: string; digest: string; imageId: string };
|
||||
source: { repository: string; ref: string; submodules: "recursive" };
|
||||
source: { repository: string; ref: string; submodules: "recursive"; cache: { retryAttempts: number; retryDelaySeconds: number } };
|
||||
modelSource: { provider: "modelscope-git-lfs"; id: string; repository: string; branch: string; ref: string; requireBranchHeadMatch: boolean };
|
||||
referenceVoice: { id: string; path: string; sha256: string; promptText: string };
|
||||
};
|
||||
@@ -160,6 +161,8 @@ function readConfig(): VoiceConfig {
|
||||
requireSha256(config.image.base.imageId, "image.base.imageId");
|
||||
requireCommit(config.image.source.ref, "image.source.ref");
|
||||
if (config.image.source.submodules !== "recursive") throw new Error(`${configLabel}.image.source.submodules must be recursive`);
|
||||
requireInteger(config.image.source.cache.retryAttempts, "image.source.cache.retryAttempts");
|
||||
requireInteger(config.image.source.cache.retryDelaySeconds, "image.source.cache.retryDelaySeconds");
|
||||
if (config.image.modelSource.provider !== "modelscope-git-lfs") throw new Error(`${configLabel}.image.modelSource.provider must be modelscope-git-lfs`);
|
||||
requireCommit(config.image.modelSource.ref, "image.modelSource.ref");
|
||||
requireString(config.image.modelSource.branch, "image.modelSource.branch");
|
||||
@@ -198,6 +201,7 @@ function readConfig(): VoiceConfig {
|
||||
}
|
||||
requireAbsolute(gpu.cacheDir, `targets.${gpu.id}.cacheDir`);
|
||||
requireAbsolute(gpu.dataDir, `targets.${gpu.id}.dataDir`);
|
||||
requireAbsolute(gpu.sourceDir, `targets.${gpu.id}.sourceDir`);
|
||||
requireInteger(config.runtime.inference.replicas, "runtime.inference.replicas");
|
||||
requireInteger(config.runtime.inference.workers, "runtime.inference.workers");
|
||||
if (config.runtime.inference.concurrency !== "serial") throw new Error(`${configLabel}.runtime.inference.concurrency must be serial`);
|
||||
@@ -316,6 +320,7 @@ function renderBundle(config: VoiceConfig): { files: Record<string, string>; sum
|
||||
containerfile: artifact(config.artifacts.containerfile),
|
||||
api: artifact(config.artifacts.api),
|
||||
modelInit: artifact(config.artifacts.modelInit),
|
||||
sourceInit: artifact(config.artifacts.sourceInit),
|
||||
d518Compose: artifact(config.artifacts.d518Compose),
|
||||
nc01Compose: artifact(config.artifacts.nc01Compose),
|
||||
frpc: replace(artifact(config.artifacts.frpcTemplate), { SERVER_ADDRESS: config.frp.server.address, CONTROL_PORT: config.frp.server.controlPort, PROXY_NAME: config.frp.tunnel.proxyName, LOCAL_ADDRESS: config.frp.tunnel.localAddress, LOCAL_PORT: config.frp.tunnel.localPort, REMOTE_PORT: config.frp.tunnel.remotePort }),
|
||||
@@ -328,7 +333,7 @@ function renderBundle(config: VoiceConfig): { files: Record<string, string>; sum
|
||||
function runtimeFiles(config: VoiceConfig, bundle: ReturnType<typeof renderBundle>, secret: ReturnType<typeof readSecret>, role: VoiceTarget["role"]): Record<string, string> {
|
||||
const target = targetByRole(config, role);
|
||||
const common = { "runtime.env": runtimeEnv(config, target), "secret.env": `${config.frp.secret.sourceKey}=${secret.value}\n` };
|
||||
return role === "frps" ? { ...common, "compose.yaml": bundle.files.nc01Compose, "frps.toml": bundle.files.frps } : { ...common, "compose.yaml": bundle.files.d518Compose, "Containerfile": bundle.files.containerfile, "api.py": bundle.files.api, "model-init.sh": bundle.files.modelInit, "frpc.toml": bundle.files.frpc };
|
||||
return role === "frps" ? { ...common, "compose.yaml": bundle.files.nc01Compose, "frps.toml": bundle.files.frps } : { ...common, "compose.yaml": bundle.files.d518Compose, "Containerfile": bundle.files.containerfile, "api.py": bundle.files.api, "model-init.sh": bundle.files.modelInit, "source-init.sh": bundle.files.sourceInit, "frpc.toml": bundle.files.frpc };
|
||||
}
|
||||
|
||||
function runtimeEnv(config: VoiceConfig, target: VoiceTarget): string {
|
||||
@@ -342,6 +347,8 @@ function runtimeEnv(config: VoiceConfig, target: VoiceTarget): string {
|
||||
SOURCE_REPOSITORY: config.image.source.repository,
|
||||
SOURCE_REF: config.image.source.ref,
|
||||
SOURCE_SUBMODULES: config.image.source.submodules,
|
||||
SOURCE_FETCH_RETRY_ATTEMPTS: config.image.source.cache.retryAttempts,
|
||||
SOURCE_FETCH_RETRY_DELAY_SECONDS: config.image.source.cache.retryDelaySeconds,
|
||||
MODEL_ID: config.image.modelSource.id,
|
||||
MODEL_REPOSITORY: config.image.modelSource.repository,
|
||||
MODEL_BRANCH: config.image.modelSource.branch,
|
||||
@@ -364,6 +371,7 @@ function runtimeEnv(config: VoiceConfig, target: VoiceTarget): string {
|
||||
VOICE_PORT: config.runtime.port,
|
||||
VOICE_CACHE_DIR: requireAbsolute(target.cacheDir, `targets.${target.id}.cacheDir`),
|
||||
VOICE_DATA_DIR: requireAbsolute(target.dataDir, `targets.${target.id}.dataDir`),
|
||||
VOICE_SOURCE_DIR: requireAbsolute(target.sourceDir, `targets.${target.id}.sourceDir`),
|
||||
VOICE_DEVICE: config.runtime.inference.device,
|
||||
VOICE_FP16: String(config.runtime.inference.fp16),
|
||||
VOICE_LOAD_TRT: String(config.runtime.inference.loadTrt),
|
||||
@@ -382,7 +390,8 @@ function runtimeEnv(config: VoiceConfig, target: VoiceTarget): string {
|
||||
|
||||
async function convergeTarget(configRoot: UniDeskConfig, config: VoiceConfig, target: VoiceTarget, files: Record<string, string>) {
|
||||
const encoded = Object.entries(files).map(([name, value]) => `printf %s ${shQuote(Buffer.from(value).toString("base64"))} | base64 -d > ${shQuote(name)}`).join("\n");
|
||||
const script = `set -eu\numask 077\nmkdir -p ${shQuote(target.workDir)}\ncd ${shQuote(target.workDir)}\n${encoded}\nchmod 600 secret.env\ndocker compose --project-name ${shQuote(target.composeProject)} --env-file runtime.env -f compose.yaml config >/dev/null\ndocker compose --project-name ${shQuote(target.composeProject)} --env-file runtime.env -f compose.yaml up -d --build\n`;
|
||||
const prepareSource = target.role === "gpu-service" ? "chmod 700 source-init.sh\nset -a\n. ./runtime.env\nset +a\n./source-init.sh\n" : "";
|
||||
const script = `set -eu\numask 077\nmkdir -p ${shQuote(target.workDir)}\ncd ${shQuote(target.workDir)}\n${encoded}\nchmod 600 secret.env\n${prepareSource}docker compose --project-name ${shQuote(target.composeProject)} --env-file runtime.env -f compose.yaml config >/dev/null\ndocker compose --project-name ${shQuote(target.composeProject)} --env-file runtime.env -f compose.yaml up -d --build\n`;
|
||||
return await capture(configRoot, target.route, ["sh"], script, { runtimeTimeoutMs: config.runtime.operations.applyTimeoutSeconds * 1000 });
|
||||
}
|
||||
|
||||
@@ -439,7 +448,7 @@ function targetByRole(config: VoiceConfig, role: VoiceTarget["role"]): VoiceTarg
|
||||
}
|
||||
|
||||
function targetSummary(target: VoiceTarget): Record<string, unknown> {
|
||||
return { id: target.id, role: target.role, route: target.route, workDir: target.workDir, composeProject: target.composeProject, cacheDir: target.cacheDir ?? null, dataDir: target.dataDir ?? null };
|
||||
return { id: target.id, role: target.role, route: target.route, workDir: target.workDir, composeProject: target.composeProject, cacheDir: target.cacheDir ?? null, dataDir: target.dataDir ?? null, sourceDir: target.sourceDir ?? null };
|
||||
}
|
||||
|
||||
function requireString(value: unknown, label: string): string {
|
||||
|
||||
Reference in New Issue
Block a user