diff --git a/config/platform-infra/selfmedia-voice.yaml b/config/platform-infra/selfmedia-voice.yaml index c30744f1..0a1248ce 100644 --- a/config/platform-infra/selfmedia-voice.yaml +++ b/config/platform-infra/selfmedia-voice.yaml @@ -17,6 +17,7 @@ artifacts: containerfile: deploy/selfmedia-voice/Containerfile api: deploy/selfmedia-voice/api.py modelInit: deploy/selfmedia-voice/model-init.sh + sourceInit: deploy/selfmedia-voice/source-init.sh d518Compose: deploy/selfmedia-voice/compose.d518.yaml nc01Compose: deploy/selfmedia-voice/compose.nc01.yaml frpcTemplate: deploy/selfmedia-voice/frpc.toml.tmpl @@ -33,6 +34,9 @@ image: repository: https://github.com/FunAudioLLM/CosyVoice.git ref: 074ca6dc9e80a2f424f1f74b48bdd7d3fea531cc submodules: recursive + cache: + retryAttempts: 3 + retryDelaySeconds: 5 modelSource: provider: modelscope-git-lfs id: FunAudioLLM/Fun-CosyVoice3-0.5B-2512 @@ -95,6 +99,7 @@ targets: workDir: /root/.unidesk/platform-infra/selfmedia-voice/runtime composeProject: unidesk-selfmedia-voice cacheDir: /root/.cache/unidesk/selfmedia-voice + sourceDir: /root/.cache/unidesk/selfmedia-voice/source dataDir: /root/.unidesk/.state/selfmedia-voice - id: NC01 role: frps diff --git a/deploy/selfmedia-voice/Containerfile b/deploy/selfmedia-voice/Containerfile index 239d75f3..e7dacb94 100644 --- a/deploy/selfmedia-voice/Containerfile +++ b/deploy/selfmedia-voice/Containerfile @@ -1,9 +1,7 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -ARG SOURCE_REPOSITORY ARG SOURCE_REF -ARG SOURCE_SUBMODULES ARG REFERENCE_WAV_PATH ARG REFERENCE_WAV_SHA256 ARG PIP_VERSION @@ -26,11 +24,10 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && python -m pip install "pip==${PIP_VERSION}" "${SETUPTOOLS_INSTALL_SPEC}" wheel -RUN git clone --no-checkout "${SOURCE_REPOSITORY}" /opt/CosyVoice \ - && git -C /opt/CosyVoice fetch --depth 1 origin "${SOURCE_REF}" \ - && git -C /opt/CosyVoice checkout --detach "${SOURCE_REF}" \ - && test "$(git -C /opt/CosyVoice rev-parse HEAD)" = "${SOURCE_REF}" \ - && if [ "${SOURCE_SUBMODULES}" = recursive ]; then git -C /opt/CosyVoice submodule update --init --recursive --depth 1; fi \ +COPY --from=cosyvoice_source . /opt/CosyVoice + +RUN test "$(git -C /opt/CosyVoice rev-parse HEAD)" = "${SOURCE_REF}" \ + && test -z "$(git -C /opt/CosyVoice submodule status --recursive | sed -n '/^[+-U]/p')" \ && printf '%s %s\n' "${REFERENCE_WAV_SHA256}" "/opt/CosyVoice/${REFERENCE_WAV_PATH}" | sha256sum -c - RUN sed -e '/^torch==/d' \ diff --git a/deploy/selfmedia-voice/compose.d518.yaml b/deploy/selfmedia-voice/compose.d518.yaml index fd9fac6d..ad902ad0 100644 --- a/deploy/selfmedia-voice/compose.d518.yaml +++ b/deploy/selfmedia-voice/compose.d518.yaml @@ -4,11 +4,11 @@ services: build: &voice-build context: . dockerfile: Containerfile + additional_contexts: + cosyvoice_source: ${VOICE_SOURCE_DIR} args: BASE_IMAGE: ${BASE_IMAGE} - SOURCE_REPOSITORY: ${SOURCE_REPOSITORY} SOURCE_REF: ${SOURCE_REF} - SOURCE_SUBMODULES: ${SOURCE_SUBMODULES} REFERENCE_WAV_PATH: ${REFERENCE_WAV_PATH} REFERENCE_WAV_SHA256: ${REFERENCE_WAV_SHA256} PIP_VERSION: ${PIP_VERSION} diff --git a/deploy/selfmedia-voice/source-init.sh b/deploy/selfmedia-voice/source-init.sh new file mode 100644 index 00000000..da3757aa --- /dev/null +++ b/deploy/selfmedia-voice/source-init.sh @@ -0,0 +1,38 @@ +#!/bin/sh +set -eu + +retry() { + attempt=1 + while ! "$@"; do + if [ "$attempt" -ge "$SOURCE_FETCH_RETRY_ATTEMPTS" ]; then + return 1 + fi + sleep "$SOURCE_FETCH_RETRY_DELAY_SECONDS" + attempt=$((attempt + 1)) + done +} + +mkdir -p "$(dirname "$VOICE_SOURCE_DIR")" +if [ ! -d "$VOICE_SOURCE_DIR/.git" ]; then + rm -rf "$VOICE_SOURCE_DIR" + mkdir -p "$VOICE_SOURCE_DIR" + git -C "$VOICE_SOURCE_DIR" init + git -C "$VOICE_SOURCE_DIR" remote add origin "$SOURCE_REPOSITORY" +fi + +test "$(git -C "$VOICE_SOURCE_DIR" remote get-url origin)" = "$SOURCE_REPOSITORY" +if ! git -C "$VOICE_SOURCE_DIR" cat-file -e "$SOURCE_REF^{commit}" 2>/dev/null; then + retry git -C "$VOICE_SOURCE_DIR" fetch --depth 1 origin "$SOURCE_REF" +fi +git -C "$VOICE_SOURCE_DIR" checkout --detach "$SOURCE_REF" +test "$(git -C "$VOICE_SOURCE_DIR" rev-parse HEAD)" = "$SOURCE_REF" + +if [ "$SOURCE_SUBMODULES" = recursive ]; then + if ! git -C "$VOICE_SOURCE_DIR" submodule status --recursive >/dev/null 2>&1 \ + || git -C "$VOICE_SOURCE_DIR" submodule status --recursive | grep -q '^[+-U]'; then + retry git -C "$VOICE_SOURCE_DIR" submodule update --init --recursive --depth 1 + fi + ! git -C "$VOICE_SOURCE_DIR" submodule status --recursive | grep -q '^[+-U]' +fi + +printf '%s %s\n' "$REFERENCE_WAV_SHA256" "$VOICE_SOURCE_DIR/$REFERENCE_WAV_PATH" | sha256sum -c - diff --git a/scripts/src/platform-infra-selfmedia-voice.ts b/scripts/src/platform-infra-selfmedia-voice.ts index 6d287b8f..f5be8ba8 100644 --- a/scripts/src/platform-infra-selfmedia-voice.ts +++ b/scripts/src/platform-infra-selfmedia-voice.ts @@ -18,6 +18,7 @@ interface VoiceTarget { composeProject: string; cacheDir?: string; dataDir?: string; + sourceDir?: string; } interface VoiceConfig { @@ -25,11 +26,11 @@ interface VoiceConfig { kind: "platform-infra-selfmedia-voice"; metadata: { id: string; owner: string; relatedIssues: number[]; mdtodo: string }; rollout: { prerequisite: string; approved: boolean }; - artifacts: Record<"root" | "containerfile" | "api" | "modelInit" | "d518Compose" | "nc01Compose" | "frpcTemplate" | "frpsTemplate", string>; + artifacts: Record<"root" | "containerfile" | "api" | "modelInit" | "sourceInit" | "d518Compose" | "nc01Compose" | "frpcTemplate" | "frpsTemplate", string>; image: { name: string; base: { repository: string; tag: string; digest: string; imageId: string }; - source: { repository: string; ref: string; submodules: "recursive" }; + source: { repository: string; ref: string; submodules: "recursive"; cache: { retryAttempts: number; retryDelaySeconds: number } }; modelSource: { provider: "modelscope-git-lfs"; id: string; repository: string; branch: string; ref: string; requireBranchHeadMatch: boolean }; referenceVoice: { id: string; path: string; sha256: string; promptText: string }; }; @@ -160,6 +161,8 @@ function readConfig(): VoiceConfig { requireSha256(config.image.base.imageId, "image.base.imageId"); requireCommit(config.image.source.ref, "image.source.ref"); if (config.image.source.submodules !== "recursive") throw new Error(`${configLabel}.image.source.submodules must be recursive`); + requireInteger(config.image.source.cache.retryAttempts, "image.source.cache.retryAttempts"); + requireInteger(config.image.source.cache.retryDelaySeconds, "image.source.cache.retryDelaySeconds"); if (config.image.modelSource.provider !== "modelscope-git-lfs") throw new Error(`${configLabel}.image.modelSource.provider must be modelscope-git-lfs`); requireCommit(config.image.modelSource.ref, "image.modelSource.ref"); requireString(config.image.modelSource.branch, "image.modelSource.branch"); @@ -198,6 +201,7 @@ function readConfig(): VoiceConfig { } requireAbsolute(gpu.cacheDir, `targets.${gpu.id}.cacheDir`); requireAbsolute(gpu.dataDir, `targets.${gpu.id}.dataDir`); + requireAbsolute(gpu.sourceDir, `targets.${gpu.id}.sourceDir`); requireInteger(config.runtime.inference.replicas, "runtime.inference.replicas"); requireInteger(config.runtime.inference.workers, "runtime.inference.workers"); if (config.runtime.inference.concurrency !== "serial") throw new Error(`${configLabel}.runtime.inference.concurrency must be serial`); @@ -316,6 +320,7 @@ function renderBundle(config: VoiceConfig): { files: Record; sum containerfile: artifact(config.artifacts.containerfile), api: artifact(config.artifacts.api), modelInit: artifact(config.artifacts.modelInit), + sourceInit: artifact(config.artifacts.sourceInit), d518Compose: artifact(config.artifacts.d518Compose), nc01Compose: artifact(config.artifacts.nc01Compose), frpc: replace(artifact(config.artifacts.frpcTemplate), { SERVER_ADDRESS: config.frp.server.address, CONTROL_PORT: config.frp.server.controlPort, PROXY_NAME: config.frp.tunnel.proxyName, LOCAL_ADDRESS: config.frp.tunnel.localAddress, LOCAL_PORT: config.frp.tunnel.localPort, REMOTE_PORT: config.frp.tunnel.remotePort }), @@ -328,7 +333,7 @@ function renderBundle(config: VoiceConfig): { files: Record; sum function runtimeFiles(config: VoiceConfig, bundle: ReturnType, secret: ReturnType, role: VoiceTarget["role"]): Record { const target = targetByRole(config, role); const common = { "runtime.env": runtimeEnv(config, target), "secret.env": `${config.frp.secret.sourceKey}=${secret.value}\n` }; - return role === "frps" ? { ...common, "compose.yaml": bundle.files.nc01Compose, "frps.toml": bundle.files.frps } : { ...common, "compose.yaml": bundle.files.d518Compose, "Containerfile": bundle.files.containerfile, "api.py": bundle.files.api, "model-init.sh": bundle.files.modelInit, "frpc.toml": bundle.files.frpc }; + return role === "frps" ? { ...common, "compose.yaml": bundle.files.nc01Compose, "frps.toml": bundle.files.frps } : { ...common, "compose.yaml": bundle.files.d518Compose, "Containerfile": bundle.files.containerfile, "api.py": bundle.files.api, "model-init.sh": bundle.files.modelInit, "source-init.sh": bundle.files.sourceInit, "frpc.toml": bundle.files.frpc }; } function runtimeEnv(config: VoiceConfig, target: VoiceTarget): string { @@ -342,6 +347,8 @@ function runtimeEnv(config: VoiceConfig, target: VoiceTarget): string { SOURCE_REPOSITORY: config.image.source.repository, SOURCE_REF: config.image.source.ref, SOURCE_SUBMODULES: config.image.source.submodules, + SOURCE_FETCH_RETRY_ATTEMPTS: config.image.source.cache.retryAttempts, + SOURCE_FETCH_RETRY_DELAY_SECONDS: config.image.source.cache.retryDelaySeconds, MODEL_ID: config.image.modelSource.id, MODEL_REPOSITORY: config.image.modelSource.repository, MODEL_BRANCH: config.image.modelSource.branch, @@ -364,6 +371,7 @@ function runtimeEnv(config: VoiceConfig, target: VoiceTarget): string { VOICE_PORT: config.runtime.port, VOICE_CACHE_DIR: requireAbsolute(target.cacheDir, `targets.${target.id}.cacheDir`), VOICE_DATA_DIR: requireAbsolute(target.dataDir, `targets.${target.id}.dataDir`), + VOICE_SOURCE_DIR: requireAbsolute(target.sourceDir, `targets.${target.id}.sourceDir`), VOICE_DEVICE: config.runtime.inference.device, VOICE_FP16: String(config.runtime.inference.fp16), VOICE_LOAD_TRT: String(config.runtime.inference.loadTrt), @@ -382,7 +390,8 @@ function runtimeEnv(config: VoiceConfig, target: VoiceTarget): string { async function convergeTarget(configRoot: UniDeskConfig, config: VoiceConfig, target: VoiceTarget, files: Record) { const encoded = Object.entries(files).map(([name, value]) => `printf %s ${shQuote(Buffer.from(value).toString("base64"))} | base64 -d > ${shQuote(name)}`).join("\n"); - const script = `set -eu\numask 077\nmkdir -p ${shQuote(target.workDir)}\ncd ${shQuote(target.workDir)}\n${encoded}\nchmod 600 secret.env\ndocker compose --project-name ${shQuote(target.composeProject)} --env-file runtime.env -f compose.yaml config >/dev/null\ndocker compose --project-name ${shQuote(target.composeProject)} --env-file runtime.env -f compose.yaml up -d --build\n`; + const prepareSource = target.role === "gpu-service" ? "chmod 700 source-init.sh\nset -a\n. ./runtime.env\nset +a\n./source-init.sh\n" : ""; + const script = `set -eu\numask 077\nmkdir -p ${shQuote(target.workDir)}\ncd ${shQuote(target.workDir)}\n${encoded}\nchmod 600 secret.env\n${prepareSource}docker compose --project-name ${shQuote(target.composeProject)} --env-file runtime.env -f compose.yaml config >/dev/null\ndocker compose --project-name ${shQuote(target.composeProject)} --env-file runtime.env -f compose.yaml up -d --build\n`; return await capture(configRoot, target.route, ["sh"], script, { runtimeTimeoutMs: config.runtime.operations.applyTimeoutSeconds * 1000 }); } @@ -439,7 +448,7 @@ function targetByRole(config: VoiceConfig, role: VoiceTarget["role"]): VoiceTarg } function targetSummary(target: VoiceTarget): Record { - return { id: target.id, role: target.role, route: target.route, workDir: target.workDir, composeProject: target.composeProject, cacheDir: target.cacheDir ?? null, dataDir: target.dataDir ?? null }; + return { id: target.id, role: target.role, route: target.route, workDir: target.workDir, composeProject: target.composeProject, cacheDir: target.cacheDir ?? null, dataDir: target.dataDir ?? null, sourceDir: target.sourceDir ?? null }; } function requireString(value: unknown, label: string): string {