update

Dockerfile

@@ -1,12 +1,14 @@
-FROM python:3.10.11-slim-bullseye
+FROM artrajz/pytorch:1.13.1-cpu-py3.10.11-ubuntu22.04
 
 RUN mkdir -p /app
 WORKDIR /app
 
 ENV DEBIAN_FRONTEND=noninteractive
 
+
 RUN apt-get update && \
-    apt-get install -yq build-essential espeak-ng cmake wget && \
+    apt-get install -yq build-essential espeak-ng cmake wget ca-certificates tzdata&& \
+    update-ca-certificates && \
     apt-get clean && \
     apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
     rm -rf /var/lib/apt/lists/* 
@@ -16,20 +18,16 @@ RUN wget https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5
     tar -xvf jemalloc-5.3.0.tar.bz2 && \
     cd jemalloc-5.3.0 && \
     ./configure && \
-    make && \
+    make -j$(nproc) && \
     make install && \
     cd .. && \
     rm -rf jemalloc-5.3.0* && \
-    ldconfig 
+    ldconfig
 
 ENV LD_PRELOAD=/usr/local/lib/libjemalloc.so
 
-RUN pip install torch --index-url https://download.pytorch.org/whl/cpu --no-cache-dir
-
 COPY requirements.txt /app/
-RUN pip install --upgrade pip && \
-    pip install pyopenjtalk==0.3.2 -i https://pypi.artrajz.cn/simple --no-cache-dir && \
-    pip install gunicorn --no-cache-dir && \
+RUN pip install gunicorn --no-cache-dir && \
     pip install -r requirements.txt --no-cache-dir&& \
     rm -rf /root/.cache/pip/*
 

Dockerfile_GPU

@@ -1,4 +1,4 @@
-FROM cnstark/pytorch:2.0.1-py3.10.11-cuda11.8.0-ubuntu22.04
+FROM artrajz/pytorch:1.13.1-cu117-py3.10.11-ubuntu22.04
 
 RUN mkdir -p /app
 WORKDIR /app
@@ -6,7 +6,7 @@ WORKDIR /app
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && \
-    apt-get install -yq build-essential espeak-ng cmake wget ca-certificates && \
+    apt-get install -yq build-essential espeak-ng cmake wget ca-certificates tzdata&& \
     update-ca-certificates && \
     apt-get clean && \
     apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
@@ -18,18 +18,16 @@ RUN wget https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5
     tar -xvf jemalloc-5.3.0.tar.bz2 && \
     cd jemalloc-5.3.0 && \
     ./configure && \
-    make && \
+    make -j$(nproc) && \
     make install && \
     cd .. && \
     rm -rf jemalloc-5.3.0* && \
-    ldconfig 
+    ldconfig
 
 ENV LD_PRELOAD=/usr/local/lib/libjemalloc.so
 
 COPY requirements.txt /app/
-RUN pip install --upgrade pip && \
-    pip install pyopenjtalk==0.3.2 fasttext -i https://pypi.artrajz.cn/simple --no-cache-dir && \
-    pip install gunicorn --no-cache-dir && \
+RUN pip install gunicorn --no-cache-dir && \
     pip install -r requirements.txt --no-cache-dir&& \
     rm -rf /root/.cache/pip/*
 

README_zh.md

@@ -21,7 +21,7 @@
 
 - [x] VITS语音合成，语音转换
 - [x] HuBert-soft VITS模型
-- [x] W2V2 VITS / emotional-vits维度情感模型
+- [x] W2V2 VITS / [emotional-vits](https://github.com/innnky/emotional-vits)维度情感模型
 - [x] [vits_chinese](https://github.com/PlayVoice/vits_chinese)
 - [x] [Bert-VITS2](https://github.com/Stardust-minus/Bert-VITS2)
 - [x] 加载多模型
@@ -341,7 +341,7 @@ pip install pyopenjtalk -i https://pypi.artrajz.cn/simple
 | 合成文本      | text      | true    |                     | str   | 需要合成语音的文本。                                         |
 | 角色id        | id        | false   | 从`config.py`中获取 | int   | 即说话人id。                                                 |
 | 音频格式      | format    | false   | 从`config.py`中获取 | str   | 支持wav,ogg,silk,mp3,flac                                    |
-| 文本语言      | lang      | false   | 从`config.py`中获取 | str   | 目前只有中文。                                               |
+| 文本语言      | lang      | false   | 从`config.py`中获取 | str   | auto为自动识别语言模式，也是默认模式，但目前只支持识别整段文本的语言，无法细分到每个句子。其余可选语言zh和ja。 |
 | 语音长度/语速 | length    | false   | 从`config.py`中获取 | float | 调节语音长度，相当于调节语速，该数值越大语速越慢。           |
 | 噪声          | noise     | false   | 从`config.py`中获取 | float | 样本噪声，控制合成的随机性。                                 |
 | sdp噪声       | noisew    | false   | 从`config.py`中获取 | float | 随机时长预测器噪声，控制音素发音长度。                       |

api_test.py

@@ -396,9 +396,9 @@ if __name__ == '__main__':
     ssml = """
     <speak lang="zh" format="mp3" length="1.2">
             <voice id="92" >这几天心里颇不宁静。</voice>
-            <voice id="125">今晚在院子里坐着乘凉，忽然想起日日走过的荷塘，在这满月的光里，总该另有一番样子吧。</voice>
+            <voice id="0" model_type="Bert-VITS2">今晚在院子里坐着乘凉，忽然想起日日走过的荷塘，在这满月的光里，总该另有一番样子吧。</voice>
             <voice id="142">月亮渐渐地升高了，墙外马路上孩子们的欢笑，已经听不见了；</voice>
-            <voice id="98">妻在屋里拍着闰儿，迷迷糊糊地哼着眠歌。</voice>
+            <voice id="0" model_type="Bert-VITS2">妻在屋里拍着闰儿，迷迷糊糊地哼着眠歌。</voice>
             <voice id="120">我悄悄地披了大衫，带上门出去。</voice><break time="2s"/>
             <voice id="121">沿着荷塘，是一条曲折的小煤屑路。</voice>
             <voice id="122">这是一条幽僻的路；白天也少人走，夜晚更加寂寞。</voice>

app.py

@@ -1,6 +1,8 @@
 import os
 import time
 import uuid
+
+from contants import ModelType
 from logger import logger
 from flask import Flask, request, send_file, jsonify, make_response, render_template
 from werkzeug.utils import secure_filename
@@ -85,28 +87,29 @@ def voice_vits_api():
         max = int(request_data.get("max", app.config.get("MAX", 50)))
         use_streaming = request_data.get('streaming', False, type=bool)
     except Exception as e:
-        logger.error(f"[VITS] {e}")
+        logger.error(f"[{ModelType.VITS.value}] {e}")
         return make_response("parameter error", 400)
 
-    logger.info(f"[VITS] id:{id} format:{format} lang:{lang} length:{length} noise:{noise} noisew:{noisew}")
-    logger.info(f"[VITS] len:{len(text)} text：{text}")
+    logger.info(
+        f"[{ModelType.VITS.value}] id:{id} format:{format} lang:{lang} length:{length} noise:{noise} noisew:{noisew}")
+    logger.info(f"[{ModelType.VITS.value}] len:{len(text)} text：{text}")
 
     if check_is_none(text):
-        logger.info(f"[VITS] text is empty")
+        logger.info(f"[{ModelType.VITS.value}] text is empty")
         return make_response(jsonify({"status": "error", "message": "text is empty"}), 400)
 
     if check_is_none(id):
-        logger.info(f"[VITS] speaker id is empty")
+        logger.info(f"[{ModelType.VITS.value}] speaker id is empty")
         return make_response(jsonify({"status": "error", "message": "speaker id is empty"}), 400)
 
     if id < 0 or id >= tts.vits_speakers_count:
-        logger.info(f"[VITS] speaker id {id} does not exist")
+        logger.info(f"[{ModelType.VITS.value}] speaker id {id} does not exist")
         return make_response(jsonify({"status": "error", "message": f"id {id} does not exist"}), 400)
 
     # 校验模型是否支持输入的语言
-    speaker_lang = tts.voice_speakers["VITS"][id].get('lang')
+    speaker_lang = tts.voice_speakers[ModelType.VITS.value][id].get('lang')
     if lang not in ["auto", "mix"] and len(speaker_lang) != 1 and lang not in speaker_lang:
-        logger.info(f"[VITS] lang \"{lang}\" is not in {speaker_lang}")
+        logger.info(f"[{ModelType.VITS.value}] lang \"{lang}\" is not in {speaker_lang}")
         return make_response(jsonify({"status": "error", "message": f"lang '{lang}' is not in {speaker_lang}"}), 400)
 
     # 如果配置文件中设置了LANGUAGE_AUTOMATIC_DETECT则强制将speaker_lang设置为LANGUAGE_AUTOMATIC_DETECT
@@ -139,10 +142,10 @@ def voice_vits_api():
         t1 = time.time()
         audio = tts.vits_infer(task)
         t2 = time.time()
-        logger.info(f"[VITS] finish in {(t2 - t1):.2f}s")
+        logger.info(f"[{ModelType.VITS.value}] finish in {(t2 - t1):.2f}s")
 
         if app.config.get("SAVE_AUDIO", False):
-            logger.debug(f"[VITS] {fname}")
+            logger.debug(f"[{ModelType.VITS.value}] {fname}")
             path = os.path.join(app.config.get('CACHE_PATH'), fname)
             save_audio(audio.getvalue(), path)
 
@@ -162,20 +165,21 @@ def voice_hubert_api():
             noisew = float(request.form.get("noisew", app.config.get("NOISEW", 0.8)))
             use_streaming = request.form.get('streaming', False, type=bool)
         except Exception as e:
-            logger.error(f"[hubert] {e}")
+            logger.error(f"[{ModelType.HUBERT_VITS.value}] {e}")
             return make_response("parameter error", 400)
 
-    logger.info(f"[hubert] id:{id} format:{format} length:{length} noise:{noise} noisew:{noisew}")
+    logger.info(
+        f"[{ModelType.HUBERT_VITS.value}] id:{id} format:{format} length:{length} noise:{noise} noisew:{noisew}")
 
     fname = secure_filename(str(uuid.uuid1()) + "." + voice.filename.split(".")[1])
     voice.save(os.path.join(app.config['UPLOAD_FOLDER'], fname))
 
     if check_is_none(id):
-        logger.info(f"[hubert] speaker id is empty")
+        logger.info(f"[{ModelType.HUBERT_VITS.value}] speaker id is empty")
         return make_response(jsonify({"status": "error", "message": "speaker id is empty"}), 400)
 
     if id < 0 or id >= tts.hubert_speakers_count:
-        logger.info(f"[hubert] speaker id {id} does not exist")
+        logger.info(f"[{ModelType.HUBERT_VITS.value}] speaker id {id} does not exist")
         return make_response(jsonify({"status": "error", "message": f"id {id} does not exist"}), 400)
 
     file_type = f"audio/{format}"
@@ -189,10 +193,10 @@ def voice_hubert_api():
     t1 = time.time()
     audio = tts.hubert_vits_infer(task)
     t2 = time.time()
-    logger.info(f"[hubert] finish in {(t2 - t1):.2f}s")
+    logger.info(f"[{ModelType.HUBERT_VITS.value}] finish in {(t2 - t1):.2f}s")
 
     if app.config.get("SAVE_AUDIO", False):
-        logger.debug(f"[hubert] {fname}")
+        logger.debug(f"[{ModelType.HUBERT_VITS.value}] {fname}")
         path = os.path.join(app.config.get('CACHE_PATH'), fname)
         save_audio(audio.getvalue(), path)
 
@@ -230,29 +234,29 @@ def voice_w2v2_api():
         emotion = int(request_data.get("emotion", app.config.get("EMOTION", 0)))
         use_streaming = request_data.get('streaming', False, type=bool)
     except Exception as e:
-        logger.error(f"[w2v2] {e}")
+        logger.error(f"[{ModelType.W2V2_VITS.value}] {e}")
         return make_response(f"parameter error", 400)
 
-    logger.info(f"[w2v2] id:{id} format:{format} lang:{lang} "
+    logger.info(f"[{ModelType.W2V2_VITS.value}] id:{id} format:{format} lang:{lang} "
                 f"length:{length} noise:{noise} noisew:{noisew} emotion:{emotion}")
-    logger.info(f"[w2v2] len:{len(text)} text：{text}")
+    logger.info(f"[{ModelType.W2V2_VITS.value}] len:{len(text)} text：{text}")
 
     if check_is_none(text):
-        logger.info(f"[w2v2] text is empty")
+        logger.info(f"[{ModelType.W2V2_VITS.value}] text is empty")
         return make_response(jsonify({"status": "error", "message": "text is empty"}), 400)
 
     if check_is_none(id):
-        logger.info(f"[w2v2] speaker id is empty")
+        logger.info(f"[{ModelType.W2V2_VITS.value}] speaker id is empty")
         return make_response(jsonify({"status": "error", "message": "speaker id is empty"}), 400)
 
     if id < 0 or id >= tts.w2v2_speakers_count:
-        logger.info(f"[w2v2] speaker id {id} does not exist")
+        logger.info(f"[{ModelType.W2V2_VITS.value}] speaker id {id} does not exist")
         return make_response(jsonify({"status": "error", "message": f"id {id} does not exist"}), 400)
 
     # 校验模型是否支持输入的语言
-    speaker_lang = tts.voice_speakers["W2V2-VITS"][id].get('lang')
+    speaker_lang = tts.voice_speakers[ModelType.W2V2_VITS.value][id].get('lang')
     if lang not in ["auto", "mix"] and len(speaker_lang) != 1 and lang not in speaker_lang:
-        logger.info(f"[w2v2] lang \"{lang}\" is not in {speaker_lang}")
+        logger.info(f"[{ModelType.W2V2_VITS.value}] lang \"{lang}\" is not in {speaker_lang}")
         return make_response(jsonify({"status": "error", "message": f"lang '{lang}' is not in {speaker_lang}"}), 400)
 
     # 如果配置文件中设置了LANGUAGE_AUTOMATIC_DETECT则强制将speaker_lang设置为LANGUAGE_AUTOMATIC_DETECT
@@ -279,10 +283,10 @@ def voice_w2v2_api():
     t1 = time.time()
     audio = tts.w2v2_vits_infer(task)
     t2 = time.time()
-    logger.info(f"[w2v2] finish in {(t2 - t1):.2f}s")
+    logger.info(f"[{ModelType.W2V2_VITS.value}] finish in {(t2 - t1):.2f}s")
 
     if app.config.get("SAVE_AUDIO", False):
-        logger.debug(f"[w2v2] {fname}")
+        logger.debug(f"[{ModelType.W2V2_VITS.value}] {fname}")
         path = os.path.join(app.config.get('CACHE_PATH'), fname)
         save_audio(audio.getvalue(), path)
 
@@ -362,7 +366,7 @@ def ssml_api():
     file_type = f"audio/{format}"
 
     t1 = time.time()
-    audio = tts.create_ssml_infer_task(voice_tasks, format)
+    audio = tts.process_ssml_infer_task(voice_tasks, format)
     t2 = time.time()
     logger.info(f"[ssml] finish in {(t2 - t1):.2f}s")
 
@@ -423,29 +427,29 @@ def voice_bert_vits2_api():
         sdp_ratio = float(request_data.get("sdp_ratio", app.config.get("SDP_RATIO", 0.2)))
         max = int(request_data.get("max", app.config.get("MAX", 50)))
     except Exception as e:
-        logger.error(f"[Bert-VITS2] {e}")
+        logger.error(f"[{ModelType.BERT_VITS2.value}] {e}")
         return make_response("parameter error", 400)
 
     logger.info(
-        f"[Bert-VITS2] id:{id} format:{format} lang:{lang} length:{length} noise:{noise} noisew:{noisew} sdp_ratio:{sdp_ratio}")
-    logger.info(f"[Bert-VITS2] len:{len(text)} text：{text}")
+        f"[{ModelType.BERT_VITS2.value}] id:{id} format:{format} lang:{lang} length:{length} noise:{noise} noisew:{noisew} sdp_ratio:{sdp_ratio}")
+    logger.info(f"[{ModelType.BERT_VITS2.value}] len:{len(text)} text：{text}")
 
     if check_is_none(text):
-        logger.info(f"[Bert-VITS2] text is empty")
+        logger.info(f"[{ModelType.BERT_VITS2.value}] text is empty")
         return make_response(jsonify({"status": "error", "message": "text is empty"}), 400)
 
     if check_is_none(id):
-        logger.info(f"[Bert-VITS2] speaker id is empty")
+        logger.info(f"[{ModelType.BERT_VITS2.value}] speaker id is empty")
         return make_response(jsonify({"status": "error", "message": "speaker id is empty"}), 400)
 
     if id < 0 or id >= tts.bert_vits2_speakers_count:
-        logger.info(f"[Bert-VITS2] speaker id {id} does not exist")
+        logger.info(f"[{ModelType.BERT_VITS2.value}] speaker id {id} does not exist")
         return make_response(jsonify({"status": "error", "message": f"id {id} does not exist"}), 400)
 
     # 校验模型是否支持输入的语言
-    speaker_lang = tts.voice_speakers["BERT-VITS2"][id].get('lang')
+    speaker_lang = tts.voice_speakers[ModelType.BERT_VITS2.value][id].get('lang')
     if lang not in ["auto", "mix"] and len(speaker_lang) != 1 and lang not in speaker_lang:
-        logger.info(f"[Bert-VITS2] lang \"{lang}\" is not in {speaker_lang}")
+        logger.info(f"[{ModelType.BERT_VITS2.value}] lang \"{lang}\" is not in {speaker_lang}")
         return make_response(jsonify({"status": "error", "message": f"lang '{lang}' is not in {speaker_lang}"}), 400)
 
     # 如果配置文件中设置了LANGUAGE_AUTOMATIC_DETECT则强制将speaker_lang设置为LANGUAGE_AUTOMATIC_DETECT
@@ -468,10 +472,10 @@ def voice_bert_vits2_api():
     t1 = time.time()
     audio = tts.bert_vits2_infer(task)
     t2 = time.time()
-    logger.info(f"[Bert-VITS2] finish in {(t2 - t1):.2f}s")
+    logger.info(f"[{ModelType.BERT_VITS2.value}] finish in {(t2 - t1):.2f}s")
 
     if app.config.get("SAVE_AUDIO", False):
-        logger.debug(f"[Bert-VITS2] {fname}")
+        logger.debug(f"[{ModelType.BERT_VITS2.value}] {fname}")
         path = os.path.join(app.config.get('CACHE_PATH'), fname)
         save_audio(audio.getvalue(), path)
 
@@ -490,18 +494,18 @@ def check():
             else:
                 request_data = request.form
 
-        model = request_data.get("model")
+        model_type_str = request_data.get("model_type", request_data.get("model")).upper()
         id = int(request_data.get("id"))
     except Exception as e:
         logger.info(f"[check] {e}")
         return make_response(jsonify({"status": "error", "message": "parameter error"}), 400)
 
-    if check_is_none(model):
-        logger.info(f"[check] model {model} is empty")
+    if check_is_none(model_type_str):
+        logger.info(f"[check] model {model_type_str} is empty")
         return make_response(jsonify({"status": "error", "message": "model is empty"}), 400)
 
-    if model.upper() not in ("VITS", "HUBERT", "W2V2"):
-        res = make_response(jsonify({"status": "error", "message": f"model {model} does not exist"}))
+    if model_type_str not in ModelType._value2member_map_:
+        res = make_response(jsonify({"status": "error", "message": f"model {model_type_str} does not exist"}))
         res.status = 404
         logger.info(f"[check] speaker id {id} error")
         return res
@@ -510,16 +514,12 @@ def check():
         logger.info(f"[check] speaker id is empty")
         return make_response(jsonify({"status": "error", "message": "speaker id is empty"}), 400)
 
-    if model.upper() == "VITS":
-        speaker_list = tts.voice_speakers["VITS"]
-    elif model.upper() == "HUBERT":
-        speaker_list = tts.voice_speakers["HUBERT-VITS"]
-    elif model.upper() == "W2V2":
-        speaker_list = tts.voice_speakers["W2V2-VITS"]
+    model_type = ModelType(model_type_str)
+    speaker_list = tts.voice_speakers[model_type.value]
 
     if len(speaker_list) == 0:
-        logger.info(f"[check] {model} not loaded")
-        return make_response(jsonify({"status": "error", "message": f"{model} not loaded"}), 400)
+        logger.info(f"[check] {model_type_str} not loaded")
+        return make_response(jsonify({"status": "error", "message": f"{model_type_str} not loaded"}), 400)
 
     if id < 0 or id >= len(speaker_list):
         logger.info(f"[check] speaker id {id} does not exist")

bert_vits2/bert_vits2.py

@@ -6,19 +6,39 @@ from bert_vits2 import utils as bert_vits2_utils
 from bert_vits2.models import SynthesizerTrn
 from bert_vits2.text import *
 from bert_vits2.text.cleaner import clean_text
+from bert_vits2.utils import process_legacy_versions
+from contants import ModelType
 from utils import classify_language, get_hparams_from_file, lang_dict
 from utils.sentence import sentence_split_and_markup, cut
 
 
 class Bert_VITS2:
     def __init__(self, model, config, device=torch.device("cpu"), **kwargs):
-        self.hps_ms = get_hparams_from_file(config)
+        self.hps_ms = get_hparams_from_file(config) if isinstance(config, str) else config
         self.n_speakers = getattr(self.hps_ms.data, 'n_speakers', 0)
         self.speakers = [item[0] for item in
                          sorted(list(getattr(self.hps_ms.data, 'spk2id', {'0': 0}).items()), key=lambda x: x[1])]
+        self.symbols = symbols
+        
+
+        # Compatible with legacy versions
+        self.version = process_legacy_versions(self.hps_ms)
+
+        if self.version in ["1.0", "1.0.0", "1.0.1"]:
+            self.symbols = symbols_legacy
+            self.hps_ms.model.n_layers_trans_flow = 3
+            
+
+        elif self.version in ["1.1.0-transition"]:
+            self.hps_ms.model.n_layers_trans_flow = 3
+
+        elif self.version in ["1.1", "1.1.0", "1.1.1"]:
+            self.hps_ms.model.n_layers_trans_flow = 6
+
+        key = f"{ModelType.BERT_VITS2.value}_v{self.version}" if self.version else ModelType.BERT_VITS2.value
+        self.lang = lang_dict.get(key, ["unknown"])
+        self.bert_handler = BertHandler(self.lang)
 
-        self.legacy = getattr(self.hps_ms.data, 'legacy', False)
-        self.symbols = symbols_legacy if self.legacy else symbols
         self._symbol_to_id = {s: i for i, s in enumerate(self.symbols)}
 
         self.net_g = SynthesizerTrn(
@@ -33,11 +53,15 @@ class Bert_VITS2:
         self.load_model(model)
 
     def load_model(self, model):
-        bert_vits2_utils.load_checkpoint(model, self.net_g, None, skip_optimizer=True)
+        bert_vits2_utils.load_checkpoint(model, self.net_g, None, skip_optimizer=True, version=self.version)
 
     def get_speakers(self):
         return self.speakers
 
+    @property
+    def sampling_rate(self):
+        return self.hps_ms.data.sampling_rate
+
     def get_text(self, text, language_str, hps):
         norm_text, phone, tone, word2ph = clean_text(text, language_str)
         phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str, self._symbol_to_id)
@@ -49,7 +73,7 @@ class Bert_VITS2:
             for i in range(len(word2ph)):
                 word2ph[i] = word2ph[i] * 2
             word2ph[0] += 1
-        bert = get_bert(norm_text, word2ph, language_str)
+        bert = self.bert_handler.get_bert(norm_text, word2ph, language_str)
         del word2ph
         assert bert.shape[-1] == len(phone), phone
 
@@ -98,7 +122,8 @@ class Bert_VITS2:
         max = voice.get("max", 50)
         # sentence_list = sentence_split_and_markup(text, max, "ZH", ["zh"])
         if lang == "auto":
-            lang = classify_language(text, target_languages=lang_dict["bert_vits2"])
+            lang = classify_language(text, target_languages=self.lang)
+
         sentence_list = cut(text, max)
         audios = []
         for sentence in sentence_list:

bert_vits2/models.py

@@ -26,9 +26,11 @@ class DurationDiscriminator(nn.Module):  # vits2
 
         self.drop = nn.Dropout(p_dropout)
         self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size, padding=kernel_size // 2)
-        # self.norm_1 = modules.LayerNorm(filter_channels)
-        self.conv_2 = nn.Conv1d(filter_channels, filter_channels, kernel_size, padding=kernel_size // 2)
-        # self.norm_2 = modules.LayerNorm(filter_channels)
+        self.norm_1 = modules.LayerNorm(filter_channels)
+        self.conv_2 = nn.Conv1d(
+            filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
+        )
+        self.norm_2 = modules.LayerNorm(filter_channels)
         self.dur_proj = nn.Conv1d(1, filter_channels, 1)
 
         self.pre_out_conv_1 = nn.Conv1d(2 * filter_channels, filter_channels, kernel_size, padding=kernel_size // 2)
@@ -36,8 +38,8 @@ class DurationDiscriminator(nn.Module):  # vits2
         self.pre_out_conv_2 = nn.Conv1d(filter_channels, filter_channels, kernel_size, padding=kernel_size // 2)
         self.pre_out_norm_2 = modules.LayerNorm(filter_channels)
 
-        # if gin_channels != 0:
-        #   self.cond = nn.Conv1d(gin_channels, in_channels, 1)
+        if gin_channels != 0:
+            self.cond = nn.Conv1d(gin_channels, in_channels, 1)
 
         self.output_layer = nn.Sequential(
             nn.Linear(filter_channels, 1),
@@ -48,13 +50,13 @@ class DurationDiscriminator(nn.Module):  # vits2
         dur = self.dur_proj(dur)
         x = torch.cat([x, dur], dim=1)
         x = self.pre_out_conv_1(x * x_mask)
-        # x = torch.relu(x)
-        # x = self.pre_out_norm_1(x)
-        # x = self.drop(x)
+        x = torch.relu(x)
+        x = self.pre_out_norm_1(x)
+        x = self.drop(x)
         x = self.pre_out_conv_2(x * x_mask)
-        # x = torch.relu(x)
-        # x = self.pre_out_norm_2(x)
-        # x = self.drop(x)
+        x = torch.relu(x)
+        x = self.pre_out_norm_2(x)
+        x = self.drop(x)
         x = x * x_mask
         x = x.transpose(1, 2)
         output_prob = self.output_layer(x)
@@ -62,17 +64,17 @@ class DurationDiscriminator(nn.Module):  # vits2
 
     def forward(self, x, x_mask, dur_r, dur_hat, g=None):
         x = torch.detach(x)
-        # if g is not None:
-        #   g = torch.detach(g)
-        #   x = x + self.cond(g)
+        if g is not None:
+            g = torch.detach(g)
+            x = x + self.cond(g)
         x = self.conv_1(x * x_mask)
-        # x = torch.relu(x)
-        # x = self.norm_1(x)
-        # x = self.drop(x)
+        x = torch.relu(x)
+        x = self.norm_1(x)
+        x = self.drop(x)
         x = self.conv_2(x * x_mask)
-        # x = torch.relu(x)
-        # x = self.norm_2(x)
-        # x = self.drop(x)
+        x = torch.relu(x)
+        x = self.norm_2(x)
+        x = self.drop(x)
 
         output_probs = []
         for dur in [dur_r, dur_hat]:
@@ -590,7 +592,7 @@ class SynthesizerTrn(nn.Module):
                  gin_channels=256,
                  use_sdp=True,
                  n_flow_layer=4,
-                 n_layers_trans_flow=3,
+                 n_layers_trans_flow=6,
                  flow_share_parameter=False,
                  use_transformer_flow=True,
                  **kwargs):

bert_vits2/text/__init__.py

@@ -1,7 +1,5 @@
 from bert_vits2.text.symbols import *
-from .chinese_bert import get_bert_feature as zh_bert
-from .english_bert_mock import get_bert_feature as en_bert
-from .japanese_bert import get_bert_feature as ja_bert
+from bert_vits2.text.bert_handler import BertHandler
 
 
 def cleaned_text_to_sequence(cleaned_text, tones, language, _symbol_to_id):
@@ -17,9 +15,3 @@ def cleaned_text_to_sequence(cleaned_text, tones, language, _symbol_to_id):
     lang_id = language_id_map[language]
     lang_ids = [lang_id for i in phones]
     return phones, tones, lang_ids
-
-
-def get_bert(norm_text, word2ph, language):
-    lang_bert_func_map = {"zh": zh_bert, "en": en_bert, "ja": ja_bert}
-    bert = lang_bert_func_map[language](norm_text, word2ph)
-    return bert

bert_vits2/text/bert_handler.py

@@ -0,0 +1,33 @@
+import importlib
+
+
+class BertHandler:
+    _bert_functions = {}
+
+    BERT_IMPORT_MAP = {
+        "zh": "bert_vits2.text.chinese_bert.get_bert_feature",
+        "en": "bert_vits2.text.english_bert_mock.get_bert_feature",
+        "ja": "bert_vits2.text.japanese_bert.get_bert_feature",
+    }
+
+    def __init__(self, languages):
+        for lang in languages:
+            if lang not in BertHandler._bert_functions:
+                self.load_bert_function(lang)
+
+    def load_bert_function(self, language):
+        if language not in BertHandler.BERT_IMPORT_MAP:
+            raise ValueError(f"Unsupported language: {language}")
+
+        module_path, function_name = BertHandler.BERT_IMPORT_MAP[language].rsplit('.', 1)
+        module = importlib.import_module(module_path, package=__package__)
+        bert_function = getattr(module, function_name)
+
+        BertHandler._bert_functions[language] = bert_function
+
+    def get_bert(self, norm_text, word2ph, language):
+        if language not in BertHandler._bert_functions:
+            raise ValueError(f"BERT for {language} has not been initialized. Please initialize first.")
+
+        bert_func = BertHandler._bert_functions[language]
+        return bert_func(norm_text, word2ph)

bert_vits2/text/cleaner.py

@@ -1,20 +1,34 @@
-from bert_vits2.text import chinese, japanese, cleaned_text_to_sequence
+import importlib
+from bert_vits2.text import cleaned_text_to_sequence
 
 language_module_map = {
-    'zh': chinese,
-    'ja': japanese
+    'zh': "bert_vits2.text.chinese",
+    'ja': "bert_vits2.text.japanese"
 }
 
+_loaded_modules = {}
+
+
+def get_language_module(language):
+    if language not in _loaded_modules:
+        module_path = language_module_map.get(language)
+        if not module_path:
+            raise ValueError(f"Unsupported language: {language}")
+
+        _loaded_modules[language] = importlib.import_module(module_path)
+
+    return _loaded_modules[language]
+
 
 def clean_text(text, language):
-    language_module = language_module_map[language]
+    language_module = get_language_module(language)
     norm_text = language_module.text_normalize(text)
     phones, tones, word2ph = language_module.g2p(norm_text)
     return norm_text, phones, tones, word2ph
 
 
 def clean_text_bert(text, language):
-    language_module = language_module_map[language]
+    language_module = get_language_module(language)
     norm_text = language_module.text_normalize(text)
     phones, tones, word2ph = language_module.g2p(norm_text)
     bert = language_module.get_bert_feature(norm_text, word2ph)

bert_vits2/utils.py

@@ -9,7 +9,7 @@ logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 logger = logging
 
 
-def load_checkpoint(checkpoint_path, model, optimizer=None, skip_optimizer=False):
+def load_checkpoint(checkpoint_path, model, optimizer=None, skip_optimizer=False, version=None):
     assert os.path.isfile(checkpoint_path)
     checkpoint_dict = torch.load(checkpoint_path, map_location='cpu')
     iteration = checkpoint_dict['iteration']
@@ -36,12 +36,19 @@ def load_checkpoint(checkpoint_path, model, optimizer=None, skip_optimizer=False
             new_state_dict[k] = saved_state_dict[k]
             assert saved_state_dict[k].shape == v.shape, (saved_state_dict[k].shape, v.shape)
         except:
-            # For upgrading from the old version
+            # Handle legacy model versions and provide appropriate warnings
             if "ja_bert_proj" in k:
                 v = torch.zeros_like(v)
+                if version is None:
+                    logger.error(f"{k} is not in the checkpoint")
+                    logger.warning(
+                        f"If you're using an older version of the model, consider adding the \"version\" parameter to the model's config.json under the \"data\" section. For instance: \"legacy_version\": \"1.0.1\"")
+            elif "flow.flows.0.enc.attn_layers.3" in k:
+                logger.error(f"{k} is not in the checkpoint")
                 logger.warning(
-                    f"If you are using an older version of the model, you should add the parameter \"legacy\":true to the data of the model's config.json")
-            logger.error(f"{k} is not in the checkpoint")
+                    f"If you're using a transitional version, please add the \"version\": \"1.1.0-transition\" parameter within the \"data\" section of the model's config.json.")
+            else:
+                logger.error(f"{k} is not in the checkpoint")
 
             new_state_dict[k] = v
     if hasattr(model, 'module'):
@@ -52,3 +59,12 @@ def load_checkpoint(checkpoint_path, model, optimizer=None, skip_optimizer=False
     logger.info("Loaded checkpoint '{}' (iteration {})".format(
         checkpoint_path, iteration))
     return model, optimizer, learning_rate, iteration
+
+
+def process_legacy_versions(hps):
+    version = getattr(hps, "version", getattr(hps.data, "version", None))
+    if version:
+        prefix = version[0].lower()
+        if prefix == "v":
+            version = version[1:]
+    return version

contants.py

@@ -0,0 +1,7 @@
+from enum import Enum
+
+class ModelType(Enum):
+    VITS = "VITS"
+    HUBERT_VITS = "HUBERT-VITS"
+    W2V2_VITS = "W2V2-VITS"
+    BERT_VITS2 = "BERT-VITS2"

docker-compose-gpu.yaml

@@ -8,14 +8,16 @@ services:
       - 23456:23456
     environment:
       LANG: 'C.UTF-8'
-      #TZ: Asia/Shanghai #timezone
+      TZ: Asia/Shanghai #timezone
+    command: ['python', 'app.py'] # 目前用不了gunicorn多进程
     volumes:
       - ./Model:/app/Model # 挂载模型文件夹
       - ./config.py:/app/config.py # 挂载配置文件
       - ./logs:/app/logs # logging logs
       - ./gunicorn_config.py:/app/gunicorn_config.py # gunicorn configuration
       - ./vits/bert:/app/vits/bert # vits_chinese
-      - ./bert_vits2/bert/chinese-roberta-wwm-ext-large:/app/bert_vits2/bert/chinese-roberta-wwm-ext-large # Bert-vits2
+      - ./bert_vits2/bert/chinese-roberta-wwm-ext-large:/app/bert_vits2/bert/chinese-roberta-wwm-ext-large # Bert-vits2 chinese
+      - ./bert_vits2/bert/bert-base-japanese-v3:/app/bert_vits2/bert/bert-base-japanese-v3 # Bert-vits2 japanese
       - ./pyopenjtalk/open_jtalk_dic_utf_8-1.11:/usr/local/lib/python3.10/site-packages/pyopenjtalk/open_jtalk_dic_utf_8-1.11 #pyopentjalk
     deploy:
       resources:

docker-compose.yaml

@@ -14,5 +14,6 @@ services:
       - ./logs:/app/logs # logging logs
       - ./gunicorn_config.py:/app/gunicorn_config.py # gunicorn configuration
       - ./vits/bert:/app/vits/bert # vits_chinese
-      - ./bert_vits2/bert/chinese-roberta-wwm-ext-large:/app/bert_vits2/bert/chinese-roberta-wwm-ext-large # Bert-vits2
+      - ./bert_vits2/bert/chinese-roberta-wwm-ext-large:/app/bert_vits2/bert/chinese-roberta-wwm-ext-large # Bert-vits2 chinese
+      - ./bert_vits2/bert/bert-base-japanese-v3:/app/bert_vits2/bert/bert-base-japanese-v3 # Bert-vits2 japanese
       - ./pyopenjtalk/open_jtalk_dic_utf_8-1.11:/usr/local/lib/python3.10/site-packages/pyopenjtalk/open_jtalk_dic_utf_8-1.11 #pyopentjalk

logger.py

@@ -22,10 +22,8 @@ handler = TimedRotatingFileHandler(log_file, when="midnight", interval=1, backup
 handler.suffix = "%Y-%m-%d.log"
 formatter = logging.Formatter('%(levelname)s:%(name)s %(message)s')
 handler.setFormatter(formatter)
-logger.addHandler(handler)
 
-logging.getLogger("werkzeug").addHandler(handler)
-logging.getLogger("apscheduler.scheduler").addHandler(handler)
+logging.getLogger().addHandler(handler)
 
 
 # Custom function to handle uncaught exceptions

utils/classify_language.py

@@ -2,16 +2,27 @@ from config import LANGUAGE_IDENTIFICATION_LIBRARY
 
 module = LANGUAGE_IDENTIFICATION_LIBRARY.lower()
 
+langid_languages = ["af", "am", "an", "ar", "as", "az", "be", "bg", "bn", "br", "bs", "ca", "cs", "cy", "da", "de", "dz", "el",
+             "en", "eo", "es", "et", "eu", "fa", "fi", "fo", "fr", "ga", "gl", "gu", "he", "hi", "hr", "ht", "hu", "hy",
+             "id", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "ku", "ky", "la", "lb", "lo", "lt", "lv", "mg",
+             "mk", "ml", "mn", "mr", "ms", "mt", "nb", "ne", "nl", "nn", "no", "oc", "or", "pa", "pl", "ps", "pt", "qu",
+             "ro", "ru", "rw", "se", "si", "sk", "sl", "sq", "sr", "sv", "sw", "ta", "te", "th", "tl", "tr", "ug", "uk",
+             "ur", "vi", "vo", "wa", "xh", "zh", "zu"]
+
 
 def classify_language(text: str, target_languages: list = None) -> str:
     if module == "fastlid" or module == "fasttext":
-        from fastlid import fastlid
+        from fastlid import fastlid, supported_langs
         classifier = fastlid
-        if target_languages != None: fastlid.set_languages = target_languages
+        if target_languages != None:
+            target_languages = [lang for lang in target_languages if lang in supported_langs]
+            fastlid.set_languages = target_languages
     elif module == "langid":
         import langid
         classifier = langid.classify
-        if target_languages != None: langid.set_languages(target_languages)
+        if target_languages != None:
+            target_languages = [lang for lang in target_languages if lang in langid_languages]
+            langid.set_languages(target_languages)
     else:
         raise ValueError(f"Wrong LANGUAGE_IDENTIFICATION_LIBRARY in config.py")
 

utils/download.py

@@ -1,3 +1,4 @@
+import logging
 import os
 import hashlib
 import tarfile
@@ -18,6 +19,7 @@ class TqdmUpTo(tqdm):
 
 
 def download_file(url, dest_path):
+    logging.info(f"Downloading: {url}")
     with TqdmUpTo(unit="B", unit_scale=True, unit_divisor=1024, miniters=1, desc=url.split('/')[-1]) as t:
         urllib.request.urlretrieve(url, dest_path, reporthook=t.update_to)
 
@@ -37,6 +39,8 @@ def extract_file(file_path, destination=None):
     if destination is None:
         destination = Path(file_path).parent
 
+    logging.info(f"Extracting to {destination}")
+
     if file_path.endswith('.zip'):
         with zipfile.ZipFile(file_path, 'r') as zip_ref:
             zip_ref.extractall(destination)
@@ -50,7 +54,7 @@ def extract_file(file_path, destination=None):
         with SevenZipFile(file_path, mode='r') as z:
             z.extractall(destination)
     else:
-        print(f"Unsupported compression format for file {file_path}")
+        logging.error(f"Unsupported compression format for file {file_path}")
 
 
 def download_and_verify(urls, target_path, expected_md5=None, extract_destination=None):

utils/lang_dict.py

@@ -1,3 +1,5 @@
+from contants import ModelType
+
 lang_dict = {
     "english_cleaners": ["en"],
     "english_cleaners2": ["en"],
@@ -18,5 +20,12 @@ lang_dict = {
                                  "ZR", "PH", "TX", "JS", "HN", "LP", "XS", "FY", "RA", "CX", "SM", "TT", "WZ", "SC",
                                  "YB"],
     "bert_chinese_cleaners": ["zh"],
-    "bert_vits2": ["zh", "ja"]
+    ModelType.BERT_VITS2.value: ["zh", "ja"],
+    f"{ModelType.BERT_VITS2.value}_v1.0": ["zh"],
+    f"{ModelType.BERT_VITS2.value}_v1.0.0": ["zh"],
+    f"{ModelType.BERT_VITS2.value}_v1.0.1": ["zh"],
+    f"{ModelType.BERT_VITS2.value}_v1.1": ["zh", "ja"],
+    f"{ModelType.BERT_VITS2.value}_v1.1.0": ["zh", "ja"],
+    f"{ModelType.BERT_VITS2.value}_v1.1.0-transition": ["zh", "ja"],
+    f"{ModelType.BERT_VITS2.value}_v1.1.1": ["zh", "ja"],
 }

utils/load_model.py

@@ -3,30 +3,33 @@ import json
 import logging
 import config
 import numpy as np
-from utils.data_utils import check_is_none
+
+import utils
+from utils.data_utils import check_is_none, HParams
 from vits import VITS
 from voice import TTS
 from config import DEVICE as device
 from utils.lang_dict import lang_dict
+from contants import ModelType
 
 
+def recognition_model_type(hps: HParams) -> str:
+    # model_config = json.load(model_config_json)
+    symbols = getattr(hps, "symbols", None)
+    # symbols = model_config.get("symbols", None)
+    emotion_embedding = getattr(hps.data, "emotion_embedding", False)
 
-def recognition_model_type(model_config_json: json) -> str:
-    model_config = json.load(model_config_json)
-    symbols = model_config.get("symbols", None)
-    emotion_embedding = model_config.get("data").get("emotion_embedding", False)
-
-    if "use_spk_conditioned_encoder" in model_config.get("model"):
-        model_type = "bert_vits2"
+    if "use_spk_conditioned_encoder" in hps.model:
+        model_type = ModelType.BERT_VITS2
         return model_type
 
     if symbols != None:
         if not emotion_embedding:
-            mode_type = "vits"
+            mode_type = ModelType.VITS
         else:
-            mode_type = "w2v2"
+            mode_type = ModelType.W2V2_VITS
     else:
-        mode_type = "hubert"
+        mode_type = ModelType.HUBERT_VITS
 
     return mode_type
 
@@ -72,16 +75,19 @@ def load_npy(emotion_reference_npy):
 
 def parse_models(model_list):
     categorized_models = {
-        "vits": [],
-        "hubert": [],
-        "w2v2": [],
-        "bert_vits2": []
+        ModelType.VITS: [],
+        ModelType.HUBERT_VITS: [],
+        ModelType.W2V2_VITS: [],
+        ModelType.BERT_VITS2: []
     }
 
     for model_info in model_list:
         config_path = model_info[1]
-        with open(config_path, 'r', encoding='utf-8') as model_config:
-            model_type = recognition_model_type(model_config)
+        hps = utils.get_hparams_from_file(config_path)
+        model_info.append(hps)
+        model_type = recognition_model_type(hps)
+        # with open(config_path, 'r', encoding='utf-8') as model_config:
+        #     model_type = recognition_model_type(model_config)
         if model_type in categorized_models:
             categorized_models[model_type].append(model_info)
 
@@ -93,23 +99,26 @@ def merge_models(model_list, model_class, model_type, additional_arg=None):
     speakers = []
     new_id = 0
 
-    for obj_id, (model_path, config_path) in enumerate(model_list):
+    for obj_id, (model_path, config_path, hps) in enumerate(model_list):
         obj_args = {
             "model": model_path,
-            "config": config_path,
+            "config": hps,
             "model_type": model_type,
             "device": device
         }
+
+        if model_type == ModelType.BERT_VITS2:
+            from bert_vits2.utils import process_legacy_versions
+            legacy_versions = process_legacy_versions(hps)
+            key = f"{model_type.value}_v{legacy_versions}" if legacy_versions else model_type.value         
+        else:
+            key = getattr(hps.data, "text_cleaners", ["none"])[0]
+
         if additional_arg:
             obj_args.update(additional_arg)
 
         obj = model_class(**obj_args)
 
-        if model_type == "bert_vits2":
-            key = model_type
-        else:
-            key = obj.get_cleaner()
-
         lang = lang_dict.get(key, ["unknown"])
 
         for real_id, name in enumerate(obj.get_speakers()):
@@ -124,11 +133,11 @@ def load_model(model_list) -> TTS:
     categorized_models = parse_models(model_list)
 
     # Handle VITS
-    vits_objs, vits_speakers = merge_models(categorized_models["vits"], VITS, "vits")
+    vits_objs, vits_speakers = merge_models(categorized_models[ModelType.VITS], VITS, ModelType.VITS)
 
     # Handle HUBERT-VITS
     hubert_vits_objs, hubert_vits_speakers = [], []
-    if len(categorized_models["hubert"]) != 0:
+    if len(categorized_models[ModelType.HUBERT_VITS]) != 0:
         if getattr(config, "HUBERT_SOFT_MODEL", None) is None or check_is_none(config.HUBERT_SOFT_MODEL):
             raise ValueError(f"Please configure HUBERT_SOFT_MODEL path in config.py")
         try:
@@ -137,13 +146,13 @@ def load_model(model_list) -> TTS:
         except Exception as e:
             raise ValueError(f"Load HUBERT_SOFT_MODEL failed {e}")
 
-        hubert_vits_objs, hubert_vits_speakers = merge_models(categorized_models["hubert"], VITS, "hubert",
+        hubert_vits_objs, hubert_vits_speakers = merge_models(categorized_models[ModelType.HUBERT_VITS], VITS, ModelType.HUBERT_VITS,
                                                               additional_arg={"additional_model": hubert})
 
     # Handle W2V2-VITS
     w2v2_vits_objs, w2v2_vits_speakers = [], []
     w2v2_emotion_count = 0
-    if len(categorized_models["w2v2"]) != 0:
+    if len(categorized_models[ModelType.W2V2_VITS]) != 0:
         if getattr(config, "DIMENSIONAL_EMOTION_NPY", None) is None or check_is_none(
                 config.DIMENSIONAL_EMOTION_NPY):
             raise ValueError(f"Please configure DIMENSIONAL_EMOTION_NPY path in config.py")
@@ -153,20 +162,24 @@ def load_model(model_list) -> TTS:
             emotion_reference = None
             raise ValueError(f"Load DIMENSIONAL_EMOTION_NPY failed {e}")
 
-        w2v2_vits_objs, w2v2_vits_speakers = merge_models(categorized_models["w2v2"], VITS, "w2v2",
+        w2v2_vits_objs, w2v2_vits_speakers = merge_models(categorized_models[ModelType.W2V2_VITS], VITS, ModelType.W2V2_VITS,
                                                           additional_arg={"additional_model": emotion_reference})
         w2v2_emotion_count = len(emotion_reference) if emotion_reference is not None else 0
 
     # Handle BERT-VITS2
     bert_vits2_objs, bert_vits2_speakers = [], []
-    if len(categorized_models["bert_vits2"]) != 0:
+    if len(categorized_models[ModelType.BERT_VITS2]) != 0:
         from bert_vits2 import Bert_VITS2
-        bert_vits2_objs, bert_vits2_speakers = merge_models(categorized_models["bert_vits2"], Bert_VITS2, "bert_vits2")
-
-    voice_obj = {"VITS": vits_objs, "HUBERT-VITS": hubert_vits_objs, "W2V2-VITS": w2v2_vits_objs,
-                 "BERT-VITS2": bert_vits2_objs}
-    voice_speakers = {"VITS": vits_speakers, "HUBERT-VITS": hubert_vits_speakers, "W2V2-VITS": w2v2_vits_speakers,
-                      "BERT-VITS2": bert_vits2_speakers}
+        bert_vits2_objs, bert_vits2_speakers = merge_models(categorized_models[ModelType.BERT_VITS2], Bert_VITS2, ModelType.BERT_VITS2)
+
+    voice_obj = {ModelType.VITS: vits_objs,
+                 ModelType.HUBERT_VITS: hubert_vits_objs,
+                 ModelType.W2V2_VITS: w2v2_vits_objs,
+                 ModelType.BERT_VITS2: bert_vits2_objs}
+    voice_speakers = {ModelType.VITS.value: vits_speakers,
+                      ModelType.HUBERT_VITS.value: hubert_vits_speakers,
+                      ModelType.W2V2_VITS.value: w2v2_vits_speakers,
+                      ModelType.BERT_VITS2.value: bert_vits2_speakers}
 
     tts = TTS(voice_obj, voice_speakers, device=device, w2v2_emotion_count=w2v2_emotion_count)
     return tts

vits-simple-api-installer-latest.sh

@@ -28,9 +28,9 @@ EN_MESSAGES=(
   ["DOWNLOAD_VITS_CHINESE"]="Do you want to download the bert model for vits_chinese? Enter 1 for yes, 2 for no."
   ["MUST_DOWNLOAD_VITS_CHINESE"]="Using vits_chinese requires downloading these models, which will take up about 410MB."
   ["DOWNLOAD_BERT_VITS2"]="Do you want to download chinese-roberta-wwm-ext-large? Enter 1 for yes or 2 for no"
-  ["MUST_DOWNLOAD_BERT_VITS2"]="To use Bert-VITS2, you must download these models, which will take up about 3.64GB."
+  ["MUST_DOWNLOAD_BERT_VITS2"]="To use Bert-VITS2, you must download these models, which will take up about 1.63GB."
   ["DOWNLOADED"]="File is downloaded correctly."
-  ["CORRUPTED"]="File is corrupted or incomplete."
+  ["CORRUPTED"]="The file may not have been downloaded, or the download might be incomplete, and it could also be corrupted."
   ["INSTALL_COMPLETE"]="The upgrade or installation has been completed."
   ["CONFIG_DIR"]="The configuration file directory is"
   ["IMPORT_NOTICE"]="If the vits model is not imported, it cannot be used. Import the model in the configuration file directory."
@@ -59,9 +59,9 @@ ZH_MESSAGES=(
   ["DOWNLOAD_VITS_CHINESE"]="是否要下载vits_chinese的bert模型？输入1表示是，2表示否。"
   ["MUST_DOWNLOAD_VITS_CHINESE"]="使用vits_chinese必须下载这些模型，将占用大约410MB。"
   ["DOWNLOAD_BERT_VITS2"]="是否要下载chinese-roberta-wwm-ext-large？输入1表示是，2表示否。"
-  ["MUST_DOWNLOAD_BERT_VITS2"]="使用Bert-VITS2必须下载这些模型，将占用大约3.64GB。"
+  ["MUST_DOWNLOAD_BERT_VITS2"]="使用Bert-VITS2必须下载这些模型，将占用大约1.63GB。"
   ["DOWNLOADED"]="文件已正确下载。"
-  ["CORRUPTED"]="文件已损坏或不完整。"
+  ["CORRUPTED"]="文件可能未下载，或下载不完整，也有可能已损坏。"
   ["INSTALL_COMPLETE"]="更新或安装已完成。"
   ["CONFIG_DIR"]="配置文件目录是"
   ["IMPORT_NOTICE"]="如果vits模型没有被导入，它是无法使用的。请在配置文件目录中导入模型。"
@@ -240,6 +240,21 @@ if [ "$choice_download_bert_vits2" -eq 1 ]; then
     download_with_fallback bert_vits2/bert/chinese-roberta-wwm-ext-large/pytorch_model.bin \
       "https://huggingface.co/hfl/chinese-roberta-wwm-ext-large/resolve/main/pytorch_model.bin"
   fi
+  
+  mkdir -p bert_vits2/bert/bert-base-japanese-v3
+
+  EXPECTED_MD5="6d0f8f3503dae04df0711b6175ef0c8e"
+  FILE_PATH="bert_vits2/bert/bert-base-japanese-v3/pytorch_model.bin"
+  echo -e "${MESSAGES["VERIFYING"]}$FILE_PATH"
+  ACTUAL_MD5=$(md5sum $FILE_PATH | awk '{print $1}')
+
+  if [ "$EXPECTED_MD5" == "$ACTUAL_MD5" ]; then
+    echo "${MESSAGES["DOWNLOADED"]}"
+  else
+    echo ${MESSAGES["CORRUPTED"]}
+    download_with_fallback bert_vits2/bert/bert-base-japanese-v3/pytorch_model.bin \
+      "https://huggingface.co/cl-tohoku/bert-base-japanese-v3/resolve/main/pytorch_model.bin"
+  fi
 
 fi
 

vits/text/japanese.py

@@ -1,7 +1,23 @@
+import os
 import re
 from unidecode import unidecode
 import pyopenjtalk
 
+from config import ABS_PATH
+from utils.download import download_and_verify
+
+URLS = [
+    "https://github.com/r9y9/open_jtalk/releases/download/v1.11.1/open_jtalk_dic_utf_8-1.11.tar.gz",
+    "https://ghproxy.com/https://github.com/r9y9/open_jtalk/releases/download/v1.11.1/open_jtalk_dic_utf_8-1.11.tar.gz",
+]
+install_path = os.path.dirname(pyopenjtalk.__file__)
+dict_path = os.path.join(install_path, "open_jtalk_dic_utf_8-1.11", "char.bin")
+TARGET_PATH = os.path.join(ABS_PATH, "open_jtalk_dic_utf_8-1.11.tar.gz")
+EXTRACT_DESTINATION = os.path.join(install_path, "")
+EXPECTED_MD5 = None
+
+if not os.path.exists(dict_path):
+    success, message = download_and_verify(URLS, TARGET_PATH, EXPECTED_MD5, EXTRACT_DESTINATION)
 
 # Regular expression matching Japanese without punctuation marks:
 _japanese_characters = re.compile(
@@ -127,7 +143,7 @@ def get_real_hatsuon(text):
 def japanese_to_ipa(text):
     text = japanese_to_romaji_with_accent(text).replace('...', '…')
     text = re.sub(
-        r'([aiueo])\1+', lambda x: x.group(0)[0]+'ː'*(len(x.group(0))-1), text)
+        r'([aiueo])\1+', lambda x: x.group(0)[0] + 'ː' * (len(x.group(0)) - 1), text)
     text = get_real_sokuon(text)
     text = get_real_hatsuon(text)
     for regex, replacement in _romaji_to_ipa:
@@ -148,6 +164,6 @@ def japanese_to_ipa3(text):
     text = japanese_to_ipa2(text).replace('n^', 'ȵ').replace(
         'ʃ', 'ɕ').replace('*', '\u0325').replace('#', '\u031a')
     text = re.sub(
-        r'([aiɯeo])\1+', lambda x: x.group(0)[0]+'ː'*(len(x.group(0))-1), text)
+        r'([aiɯeo])\1+', lambda x: x.group(0)[0] + 'ː' * (len(x.group(0)) - 1), text)
     text = re.sub(r'((?:^|\s)(?:ts|tɕ|[kpt]))', r'\1ʰ', text)
     return text

vits/vits.py

@@ -4,6 +4,8 @@ import numpy as np
 import torch
 from torch import no_grad, LongTensor, inference_mode, FloatTensor
 import utils
+from contants import ModelType
+from utils import get_hparams_from_file, lang_dict
 from utils.sentence import sentence_split_and_markup
 from vits import commons
 from vits.mel_processing import spectrogram_torch
@@ -11,11 +13,10 @@ from vits.text import text_to_sequence
 from vits.models import SynthesizerTrn
 
 
-
 class VITS:
-    def __init__(self, model, config, additional_model=None, model_type=None, device=torch.device("cpu"),**kwargs):
+    def __init__(self, model, config, additional_model=None, model_type=None, device=torch.device("cpu"), **kwargs):
         self.model_type = model_type
-        self.hps_ms = utils.get_hparams_from_file(config)
+        self.hps_ms = get_hparams_from_file(config) if isinstance(config, str) else config
         self.n_speakers = getattr(self.hps_ms.data, 'n_speakers', 0)
         self.n_symbols = len(getattr(self.hps_ms, 'symbols', []))
         self.speakers = getattr(self.hps_ms, 'speakers', ['0'])
@@ -38,15 +39,18 @@ class VITS:
         _ = self.net_g_ms.eval()
         self.device = device
 
+        key = getattr(self.hps_ms.data, "text_cleaners", ["none"])[0]
+        self.lang = lang_dict.get(key, ["unknown"])
+
         # load model
         self.load_model(model, additional_model)
 
     def load_model(self, model, additional_model=None):
         utils.load_checkpoint(model, self.net_g_ms)
         self.net_g_ms.to(self.device)
-        if self.model_type == "hubert":
+        if self.model_type == ModelType.HUBERT_VITS:
             self.hubert = additional_model
-        elif self.model_type == "w2v2":
+        elif self.model_type == ModelType.W2V2_VITS:
             self.emotion_reference = additional_model
 
     def get_cleaned_text(self, text, hps, cleaned=False):
@@ -71,13 +75,17 @@ class VITS:
     def get_speakers(self, escape=False):
         return self.speakers
 
+    @property
+    def sampling_rate(self):
+        return self.hps_ms.data.sampling_rate
+
     def infer(self, params):
         with no_grad():
             x_tst = params.get("stn_tst").unsqueeze(0).to(self.device)
             x_tst_lengths = LongTensor([params.get("stn_tst").size(0)]).to(self.device)
             x_tst_prosody = torch.FloatTensor(params.get("char_embeds")).unsqueeze(0).to(
                 self.device) if self.bert_embedding else None
-            sid = params.get("sid").to(self.device) if not self.bert_embedding else None
+            sid = params.get("sid").to(self.device)
             emotion = params.get("emotion").to(self.device) if self.emotion_embedding else None
 
             audio = self.net_g_ms.infer(x=x_tst,
@@ -97,15 +105,14 @@ class VITS:
                         emotion=None, cleaned=False, f0_scale=1):
         emo = None
         char_embeds = None
-        if self.model_type != "hubert":
+        if self.model_type != ModelType.HUBERT_VITS:
             if self.bert_embedding:
                 stn_tst, char_embeds = self.get_cleaned_text(text, self.hps_ms, cleaned=cleaned)
-                sid = None
             else:
                 stn_tst = self.get_cleaned_text(text, self.hps_ms, cleaned=cleaned)
-                sid = LongTensor([speaker_id])
+            sid = LongTensor([speaker_id])
 
-        if self.model_type == "w2v2":
+        if self.model_type == ModelType.W2V2_VITS:
             # if emotion_reference.endswith('.npy'):
             #     emotion = np.load(emotion_reference)
             #     emotion = FloatTensor(emotion).unsqueeze(0)
@@ -121,7 +128,7 @@ class VITS:
             emo = torch.FloatTensor(self.emotion_reference[emotion]).unsqueeze(0)
 
 
-        elif self.model_type == "hubert":
+        elif self.model_type == ModelType.HUBERT_VITS:
             if self.use_f0:
                 audio, sampling_rate = librosa.load(audio_path, sr=self.hps_ms.data.sampling_rate, mono=True)
                 audio16000 = librosa.resample(audio, orig_sr=sampling_rate, target_sr=16000)
@@ -165,35 +172,39 @@ class VITS:
         if text is not None: text = re.sub(r'\s+', ' ', text).strip()
 
         tasks = []
-        if self.model_type == "vits":
+        if self.model_type == ModelType.VITS:
             sentence_list = sentence_split_and_markup(text, max, lang, speaker_lang)
             for sentence in sentence_list:
                 params = self.get_infer_param(text=sentence, speaker_id=speaker_id, length_scale=length,
                                               noise_scale=noise, noise_scale_w=noisew)
                 tasks.append(params)
 
-        elif self.model_type == "hubert":
+        elif self.model_type == ModelType.HUBERT_VITS:
             params = self.get_infer_param(speaker_id=speaker_id, length_scale=length, noise_scale=noise,
                                           noise_scale_w=noisew, audio_path=audio_path)
             tasks.append(params)
 
-        elif self.model_type == "w2v2":
+        elif self.model_type == ModelType.W2V2_VITS:
             sentence_list = sentence_split_and_markup(text, max, lang, speaker_lang)
             for sentence in sentence_list:
                 params = self.get_infer_param(text=sentence, speaker_id=speaker_id, length_scale=length,
                                               noise_scale=noise, noise_scale_w=noisew, emotion=emotion)
                 tasks.append(params)
+        else:
+            raise ValueError(f"Unsupported model type: {self.model_type}")
 
         return tasks
 
     def get_audio(self, voice, auto_break=False):
         tasks = self.get_tasks(voice)
         # 停顿0.75s，避免语音分段合成再拼接后的连接突兀
-        brk = np.zeros(int(0.75 * 22050), dtype=np.int16)
+        brk = np.zeros(int(0.75 * self.sampling_rate), dtype=np.int16)
 
         audios = []
-        for task in tasks:
-            if auto_break:
+        num_tasks = len(tasks)
+
+        for i, task in enumerate(tasks):
+            if auto_break and i < num_tasks - 1:
                 chunk = np.concatenate((self.infer(task), brk), axis=0)
             else:
                 chunk = self.infer(task)

voice.py

@@ -8,8 +8,9 @@ import config
 import soundfile as sf
 from io import BytesIO
 from graiax import silkcoder
-import utils
 from logger import logger
+from contants import ModelType
+from scipy.signal import resample_poly
 
 
 # torch.set_num_threads(1) # 设置torch线程为1
@@ -21,11 +22,11 @@ class TTS:
         self._voice_speakers = voice_speakers
         self._strength_dict = {"x-weak": 0.25, "weak": 0.5, "Medium": 0.75, "Strong": 1, "x-strong": 1.25}
         self._speakers_count = sum([len(self._voice_speakers[i]) for i in self._voice_speakers])
-        self._vits_speakers_count = len(self._voice_speakers["VITS"])
-        self._hubert_speakers_count = len(self._voice_speakers["HUBERT-VITS"])
-        self._w2v2_speakers_count = len(self._voice_speakers["W2V2-VITS"])
+        self._vits_speakers_count = len(self._voice_speakers[ModelType.VITS.value])
+        self._hubert_speakers_count = len(self._voice_speakers[ModelType.HUBERT_VITS.value])
+        self._w2v2_speakers_count = len(self._voice_speakers[ModelType.W2V2_VITS.value])
         self._w2v2_emotion_count = kwargs.get("w2v2_emotion_count", 0)
-        self._bert_vits2_speakers_count = len(self._voice_speakers["BERT-VITS2"])
+        self._bert_vits2_speakers_count = len(self._voice_speakers[ModelType.BERT_VITS2.value])
         self.dem = None
 
         # Initialization information
@@ -42,11 +43,11 @@ class TTS:
             except Exception as e:
                 self.logger.warning(f"Load DIMENSIONAL_EMOTION_MODEL failed {e}")
 
-        if self._vits_speakers_count != 0: self.logger.info(f"[VITS] {self._vits_speakers_count} speakers")
-        if self._hubert_speakers_count != 0: self.logger.info(f"[hubert] {self._hubert_speakers_count} speakers")
-        if self._w2v2_speakers_count != 0: self.logger.info(f"[w2v2] {self._w2v2_speakers_count} speakers")
+        if self._vits_speakers_count != 0: self.logger.info(f"[{ModelType.VITS.value}] {self._vits_speakers_count} speakers")
+        if self._hubert_speakers_count != 0: self.logger.info(f"[{ModelType.HUBERT_VITS.value}] {self._hubert_speakers_count} speakers")
+        if self._w2v2_speakers_count != 0: self.logger.info(f"[{ModelType.W2V2_VITS.value}] {self._w2v2_speakers_count} speakers")
         if self._bert_vits2_speakers_count != 0: self.logger.info(
-            f"[Bert-VITS2] {self._bert_vits2_speakers_count} speakers")
+            f"[{ModelType.BERT_VITS2.value}] {self._bert_vits2_speakers_count} speakers")
         self.logger.info(f"{self._speakers_count} speakers in total.")
         if self._speakers_count == 0:
             self.logger.warning(f"No model was loaded.")
@@ -124,6 +125,15 @@ class TTS:
                 break
             yield chunk
 
+    def resample_audio(self, audio, orig_sr, target_sr):
+        if orig_sr == target_sr:
+            return audio
+
+        gcd = np.gcd(orig_sr, target_sr)
+        audio = resample_poly(audio, target_sr // gcd, orig_sr // gcd)
+
+        return audio
+
     def parse_ssml(self, ssml):
         root = ET.fromstring(ssml)
         format = root.attrib.get("format", "wav")
@@ -140,9 +150,11 @@ class TTS:
                 noisew = float(element.attrib.get("noisew", root.attrib.get("noisew", config.NOISEW)))
                 max = int(element.attrib.get("max", root.attrib.get("max", "0")))
                 # 不填写默认就是vits
-                model = element.attrib.get("model", root.attrib.get("model", "vits"))
+                model_type = element.attrib.get("model_type", root.attrib.get("model_type", "vits"))
                 # w2v2-vits/emotion-vits才有emotion
                 emotion = int(element.attrib.get("emotion", root.attrib.get("emotion", 0)))
+                # Bert-VITS2的参数
+                sdp_ratio = int(element.attrib.get("sdp_ratio", root.attrib.get("sdp_ratio", config.SDP_RATIO)))
 
                 voice_element = ET.tostring(element, encoding='unicode')
 
@@ -177,8 +189,9 @@ class TTS:
                                             "noise": noise,
                                             "noisew": noisew,
                                             "max": max,
-                                            "model": model,
-                                            "emotion": emotion
+                                            "model_type": model_type,
+                                            "emotion": emotion,
+                                            "sdp_ratio": sdp_ratio
                                             })
 
                 # 分段末尾停顿0.75s
@@ -197,39 +210,49 @@ class TTS:
 
         return voice_tasks, format
 
-    def create_ssml_infer_task(self, tasks, format):
+    def process_ssml_infer_task(self, tasks, format):
         audios = []
+        sampling_rates = []
+        last_sampling_rate = 22050
         for task in tasks:
             if task.get("break"):
                 audios.append(np.zeros(int(task.get("break") * 22050), dtype=np.int16))
+                sampling_rates.append(last_sampling_rate)
             else:
-                model = task.get("model").upper()
-                if model != "VITS" and model != "W2V2-VITS" and model != "EMOTION-VITS":
-                    raise ValueError(f"Unsupported model: {task.get('model')}")
-                voice_obj = self._voice_obj[model][task.get("id")][1]
-                task["id"] = self._voice_obj[model][task.get("id")][0]
+                model_type_str = task.get("model_type").upper()
+                if model_type_str not in [ModelType.VITS.value, ModelType.W2V2_VITS.value, ModelType.BERT_VITS2.value]:
+                    raise ValueError(f"Unsupported model type: {task.get('model_type')}")
+                model_type = ModelType(model_type_str)
+                voice_obj = self._voice_obj[model_type][task.get("id")][1]
+                real_id = self._voice_obj[model_type][task.get("id")][0]
+                task["id"] = real_id
+                sampling_rates.append(voice_obj.sampling_rate)
+                last_sampling_rate = voice_obj.sampling_rate
                 audio = voice_obj.get_audio(task)
                 audios.append(audio)
-
-        audio = np.concatenate(audios, axis=0)
-        encoded_audio = self.encode(voice_obj.hps_ms.data.sampling_rate, audio, format)
+        # 得到最高的采样率
+        target_sr = max(sampling_rates)
+        # 所有音频要与最高采样率保持一致
+        resampled_audios = [self.resample_audio(audio, sr, target_sr) for audio, sr in zip(audios, sampling_rates)]
+        audio = np.concatenate(resampled_audios, axis=0)
+        encoded_audio = self.encode(target_sr, audio, format)
         return encoded_audio
 
     def vits_infer(self, task):
         format = task.get("format", "wav")
-        voice_obj = self._voice_obj["VITS"][task.get("id")][1]
-        real_id = self._voice_obj["VITS"][task.get("id")][0]
+        voice_obj = self._voice_obj[ModelType.VITS][task.get("id")][1]
+        real_id = self._voice_obj[ModelType.VITS][task.get("id")][0]
         task["id"] = real_id  # Change to real id
-        sampling_rate = voice_obj.hps_ms.data.sampling_rate
+        sampling_rate = voice_obj.sampling_rate
         audio = voice_obj.get_audio(task, auto_break=True)
         encoded_audio = self.encode(sampling_rate, audio, format)
         return encoded_audio
 
     def stream_vits_infer(self, task, fname=None):
         format = task.get("format", "wav")
-        voice_obj = self._voice_obj["VITS"][task.get("id")][1]
-        task["id"] = self._voice_obj["VITS"][task.get("id")][0]
-        sampling_rate = voice_obj.hps_ms.data.sampling_rate
+        voice_obj = self._voice_obj[ModelType.VITS][task.get("id")][1]
+        task["id"] = self._voice_obj[ModelType.VITS][task.get("id")][0]
+        sampling_rate = voice_obj.sampling_rate
         genertator = voice_obj.get_stream_audio(task, auto_break=True)
         # audio = BytesIO()
         for chunk in genertator:
@@ -244,18 +267,18 @@ class TTS:
 
     def hubert_vits_infer(self, task):
         format = task.get("format", "wav")
-        voice_obj = self._voice_obj["HUBERT-VITS"][task.get("id")][1]
-        task["id"] = self._voice_obj["HUBERT-VITS"][task.get("id")][0]
-        sampling_rate = voice_obj.hps_ms.data.sampling_rate
+        voice_obj = self._voice_obj[ModelType.HUBERT_VITS][task.get("id")][1]
+        task["id"] = self._voice_obj[ModelType.HUBERT_VITS][task.get("id")][0]
+        sampling_rate = voice_obj.sampling_rate
         audio = voice_obj.get_audio(task)
         encoded_audio = self.encode(sampling_rate, audio, format)
         return encoded_audio
 
     def w2v2_vits_infer(self, task):
         format = task.get("format", "wav")
-        voice_obj = self._voice_obj["W2V2-VITS"][task.get("id")][1]
-        task["id"] = self._voice_obj["W2V2-VITS"][task.get("id")][0]
-        sampling_rate = voice_obj.hps_ms.data.sampling_rate
+        voice_obj = self._voice_obj[ModelType.W2V2_VITS][task.get("id")][1]
+        task["id"] = self._voice_obj[ModelType.W2V2_VITS][task.get("id")][0]
+        sampling_rate = voice_obj.sampling_rate
         audio = voice_obj.get_audio(task, auto_break=True)
         encoded_audio = self.encode(sampling_rate, audio, format)
         return encoded_audio
@@ -265,17 +288,17 @@ class TTS:
         target_id = task.get("target_id")
         format = task.get("format")
 
-        original_id_obj = int(self._voice_obj["VITS"][original_id][2])
-        target_id_obj = int(self._voice_obj["VITS"][target_id][2])
+        original_id_obj = int(self._voice_obj[ModelType.VITS][original_id][2])
+        target_id_obj = int(self._voice_obj[ModelType.VITS][target_id][2])
 
         if original_id_obj != target_id_obj:
             raise ValueError(f"speakers are in diffrent VITS Model")
 
-        task["original_id"] = int(self._voice_obj["VITS"][original_id][0])
-        task["target_id"] = int(self._voice_obj["VITS"][target_id][0])
+        task["original_id"] = int(self._voice_obj[ModelType.VITS][original_id][0])
+        task["target_id"] = int(self._voice_obj[ModelType.VITS][target_id][0])
 
-        voice_obj = self._voice_obj["VITS"][original_id][1]
-        sampling_rate = voice_obj.hps_ms.data.sampling_rate
+        voice_obj = self._voice_obj[ModelType.VITS][original_id][1]
+        sampling_rate = voice_obj.sampling_rate
 
         audio = voice_obj.voice_conversion(task)
         encoded_audio = self.encode(sampling_rate, audio, format)
@@ -294,9 +317,9 @@ class TTS:
 
     def bert_vits2_infer(self, task):
         format = task.get("format", "wav")
-        voice_obj = self._voice_obj["BERT-VITS2"][task.get("id")][1]
-        task["id"] = self._voice_obj["BERT-VITS2"][task.get("id")][0]
-        sampling_rate = voice_obj.hps_ms.data.sampling_rate
+        voice_obj = self._voice_obj[ModelType.BERT_VITS2][task.get("id")][1]
+        task["id"] = self._voice_obj[ModelType.BERT_VITS2][task.get("id")][0]
+        sampling_rate = voice_obj.sampling_rate
         audio = voice_obj.get_audio(task, auto_break=True)
         encoded_audio = self.encode(sampling_rate, audio, format)
         return encoded_audio