Update app.py

app.py

@@ -15,7 +15,8 @@ logging.basicConfig(
 )
 
 logger = logging.getLogger(__name__)
-
+import shutil
+from scipy.io.wavfile import write
 import librosa
 import numpy as np
 import torch
@@ -23,7 +24,7 @@ import torch.nn as nn
 from torch.utils.data import Dataset
 from torch.utils.data import DataLoader, Dataset
 from tqdm import tqdm
-from clap_wrapper import get_clap_audio_feature, get_clap_text_feature
+from tools.sentence import extrac, is_japanese, is_chinese, seconds_to_ass_time, extract_text_from_file, remove_annotations,extract_and_convert
 
 
 import gradio as gr
@@ -40,7 +41,7 @@ import utils
 from models import SynthesizerTrn
 from text.symbols import symbols
 import sys
-
+import re
 from tools.translate import translate
 
 net_g = None
@@ -132,9 +133,10 @@ def infer(
     sid,
     style_text=None,
     style_weight=0.7,
+    language = "Auto",
 ):
-
-    language= 'JP' if is_japanese(text) else 'ZH'
+    if language == "Auto":
+        language= 'JP' if is_japanese(text) else 'ZH'
     bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
         text,
         language,
@@ -198,6 +200,146 @@ def loadmodel(model):
     _ = utils.load_checkpoint(model, net_g, None, skip_optimizer=True)
     return "success"
 
+def generate_audio_and_srt_for_group(group, outputPath, group_index, sampling_rate, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale,spealerList,silenceTime):
+    audio_fin = []
+    ass_entries = []
+    start_time = 0
+    #speaker = random.choice(cara_list)
+    ass_header = """[Script Info]
+; 我没意见
+Title: Audiobook
+ScriptType: v4.00+
+WrapStyle: 0
+PlayResX: 640
+PlayResY: 360
+ScaledBorderAndShadow: yes
+[V4+ Styles]
+Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
+Style: Default,Arial,20,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,0,0,0,0,100,100,0,0,1,1,1,2,10,10,10,1
+[Events]
+Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
+"""
+
+    for sentence in group:
+        try:
+            FakeSpeaker = sentence.split("|")[0]
+            print(FakeSpeaker)
+            SpeakersList = re.split('\n', spealerList)
+            if FakeSpeaker in list(hps.data.spk2id.keys()):
+                speaker = FakeSpeaker
+            for i in SpeakersList:
+                if FakeSpeaker == i.split("|")[1]:
+                    speaker = i.split("|")[0]
+            if sentence != '\n':
+                audio = infer_simple((remove_annotations(sentence.split("|")[-1]).replace(" ","")+"。").replace("，。","。").replace("。。","。"), sdp_ratio, noise_scale, noise_scale_w, length_scale,speaker)
+                silence_frames = int(silenceTime * 44010) if is_chinese(sentence) else int(silenceTime * 44010)
+                silence_data = np.zeros((silence_frames,), dtype=audio.dtype)
+                audio_fin.append(audio)
+                audio_fin.append(silence_data)
+
+                duration = len(audio) / sampling_rate
+                print(duration)
+                end_time = start_time + duration + silenceTime
+                ass_entries.append("Dialogue: 0,{},{},".format(seconds_to_ass_time(start_time), seconds_to_ass_time(end_time)) + "Default,,0,0,0,,{}".format(sentence.replace("|","：")))
+                start_time = end_time
+        except:
+            pass
+    wav_filename = os.path.join(outputPath, f'audiobook_part_{group_index}.wav')
+    ass_filename = os.path.join(outputPath, f'audiobook_part_{group_index}.ass')
+
+    write(wav_filename, sampling_rate, np.concatenate(audio_fin))
+
+    with open(ass_filename, 'w', encoding='utf-8') as f:
+        f.write(ass_header + '\n'.join(ass_entries))
+    return (hps.data.sampling_rate, np.concatenate(audio_fin))
+
+def audiobook(inputFile, groupsize, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale,spealerList,silenceTime,filepath,raw_text):
+    directory_path = filepath if torch.cuda.is_available() else "books"
+ 
+    if os.path.exists(directory_path):
+        shutil.rmtree(directory_path)
+
+    os.makedirs(directory_path)
+    if inputFile:
+        text = extract_text_from_file(inputFile.name)
+    else:
+        text = raw_text
+    sentences = extrac(extract_and_convert(text))
+    GROUP_SIZE = groupsize
+    for i in range(0, len(sentences), GROUP_SIZE):
+        group = sentences[i:i+GROUP_SIZE]
+        if spealerList == "":
+            spealerList = "无"
+        result = generate_audio_and_srt_for_group(group,directory_path, i//GROUP_SIZE + 1, 44100, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale,spealerList,silenceTime)
+        if not torch.cuda.is_available():
+            return result
+    return result
+
+def infer_simple(
+    text,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    sid,
+    style_text=None,
+    style_weight=0.7,
+):
+    if is_chinese(text) or is_japanese(text):
+        if len(text) > 1:
+            language= 'JP' if is_japanese(text) else 'ZH'
+            bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
+                text,
+                language,
+                hps,
+                device,
+                style_text="",
+                style_weight=0,
+            )
+            with torch.no_grad():
+                x_tst = phones.to(device).unsqueeze(0)
+                tones = tones.to(device).unsqueeze(0)
+                lang_ids = lang_ids.to(device).unsqueeze(0)
+                bert = bert.to(device).unsqueeze(0)
+                ja_bert = ja_bert.to(device).unsqueeze(0)
+                en_bert = en_bert.to(device).unsqueeze(0)
+                x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
+                # emo = emo.to(device).unsqueeze(0)
+                del phones
+                speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
+                audio = (
+                    net_g.infer(
+                        x_tst,
+                        x_tst_lengths,
+                        speakers,
+                        tones,
+                        lang_ids,
+                        bert,
+                        ja_bert,
+                        en_bert,
+                        sdp_ratio=sdp_ratio,
+                        noise_scale=noise_scale,
+                        noise_scale_w=noise_scale_w,
+                        length_scale=length_scale,
+                    )[0][0, 0]
+                    .data.cpu()
+                    .float()
+                    .numpy()
+                )
+                del (
+                    x_tst,
+                    tones,
+                    lang_ids,
+                    bert,
+                    x_tst_lengths,
+                    speakers,
+                    ja_bert,
+                    en_bert,
+                )  # , emo
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+                return audio
+
 if __name__ == "__main__":
     languages = [ "Auto", "ZH", "JP"]
     modelPaths = []
@@ -206,7 +348,7 @@ if __name__ == "__main__":
             modelPaths.append(os.path.join(dirpath, filename))
     hps = utils.get_hparams_from_file('Data/BangDream/configs/config.json')
     net_g = get_net_g(
-        model_path=modelPaths[-1], device=device, hps=hps
+        model_path="Data/BangDream/models/G_1536000.pth", device=device, hps=hps
     )
     speaker_ids = hps.data.spk2id
     speakers = list(speaker_ids.keys())
@@ -217,7 +359,7 @@ if __name__ == "__main__":
             [好玩的](http://love.soyorin.top/)\n
             该界面的真实链接(国内可用): https://mahiruoshi-bangdream-bert-vits2.hf.space/\n
             API: https://mahiruoshi-bert-vits2-api.hf.space/ \n
-            调用方式: https://mahiruoshi-bert-vits2-api.hf.space/?text=%E4%B8%BA%E4%BB%80%E4%B9%88%E8%A6%81%E6%BC%94%E5%A5%8F%E6%98%A5%E6%97%A5%E5%BD%B1&speaker=%E9%A6%99%E6%BE%84\n
+            调用方式: https://mahiruoshi-bert-vits2-api.hf.space/?text={{speakText}}&speaker=chosen_speaker\n
             推荐搭配[Legado开源阅读](https://github.com/gedoor/legado)或[聊天bot](https://github.com/Paraworks/BangDreamAi)使用\n
             二创请标注作者：B站@Mahiroshi: https://space.bilibili.com/19874615\n
             训练数据集归属：BangDream及少歌手游,提取自BestDori,[数据集获取流程](https://nijigaku.top/2023/09/29/Bestbushiroad%E8%AE%A1%E5%88%92-vits-%E9%9F%B3%E9%A2%91%E6%8A%93%E5%8F%96%E5%8F%8A%E6%95%B0%E6%8D%AE%E9%9B%86%E5%AF%B9%E9%BD%90/)\n
@@ -238,6 +380,9 @@ if __name__ == "__main__":
                                 length_scale = gr.Slider(
                                         minimum=0.1, maximum=2, value=1, step=0.01, label="语速调节"
                                     )
+                                language = gr.Dropdown(
+                                        choices=languages, value="Auto", label="语言"
+                                    )
                                 with gr.Accordion(label="参数设定", open=True):
                                     sdp_ratio = gr.Slider(
                                     minimum=0, maximum=1, value=0.5, step=0.01, label="SDP/DP混合比"
@@ -262,7 +407,12 @@ if __name__ == "__main__":
                                     info="输入纯日语或者中文",
                                     value="我是来结束这个乐队的。",
                                 )
-                                style_text = gr.Textbox(label="辅助文本",info="语言保持跟主文本一致",placeholder="为什么要演奏春日影!")
+                                style_text = gr.Textbox(
+                                    label="情感辅助文本",
+                                    info="语言保持跟主文本一致,文本可以参考训练集:https://huggingface.co/spaces/Mahiruoshi/BangDream-Bert-VITS2/blob/main/filelists/Mygo.list)",
+                                    placeholder="使用辅助文本的语意来辅助生成对话（语���保持与主文本相同）\n\n"
+                        "**注意**：不要使用**指令式文本**（如：开心），要使用**带有强烈情感的文本**（如：我好快乐！！！）"
+                                                        )
                                 style_weight = gr.Slider(
                                         minimum=0,
                                         maximum=1,
@@ -274,7 +424,7 @@ if __name__ == "__main__":
                                 btn = gr.Button("点击生成", variant="primary")
                                 audio_output = gr.Audio(label="Output Audio")
                                 btntran = gr.Button("快速中翻日")
-                                translateResult = gr.TextArea(label="百度翻译",value="从这里翻译后的文本")
+                                translateResult = gr.TextArea(label="使用百度翻译",placeholder="从这里复制翻译后的文本")
                                 btntran.click(translate, inputs=[text], outputs = [translateResult])
                                 
                     btn.click(
@@ -288,14 +438,76 @@ if __name__ == "__main__":
                             speaker,
                             style_text,
                             style_weight,
+                            language,
                         ],
                         outputs=[audio_output],
                     )
         with gr.TabItem('少歌在2.2版本'):
-            gr.Markdown(
-                '<div align="center">'
-                f'<iframe style="width:100%;height:400px;" src="https://mahiruoshi-mygo-vits-bert.hf.space/" frameborder="0"></iframe>'
-                '</div>'
+            gr.Markdown(value="""
+                <div align="center">
+                <iframe style="width:100%;height:400px;" src="https://mahiruoshi-mygo-vits-bert.hf.space/" frameborder="0"></iframe>'
+                </div>"""
             )
+        with gr.Tab('拓展功能'):
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown(
+                                    f"从 <a href='https://nijigaku.top/2023/10/03/BangDreamTTS/'>我的博客站点</a> 查看自制galgame使用说明\n</a>"
+                                )
+                    inputFile = gr.UploadButton(label="txt文件输入")
+                    raw_text = gr.TextArea(
+                                        label="文本输入",
+                                        info="输入纯日语或者中文",
+                                        value="つくし|我是来结束这个乐队的。",
+                    )
+                    groupSize = gr.Slider(
+                    minimum=10, maximum=1000 if  torch.cuda.is_available() else 50,value = 50, step=1, label="单个音频文件包含的最大字数"
+                    )
+                    silenceTime = gr.Slider(
+                    minimum=0, maximum=1, value=0.5, step=0.01, label="句子的间隔"
+                    )
+                    filepath = gr.TextArea(
+                                        label="本地合成时的音频存储文件夹(会清空文件夹)",
+                                        value = "D:/audiobook/book1",
+                    )
+                    spealerList = gr.TextArea(
+                                        label="角色对应表,左边是你想要在每一句话合成中用到的speaker(见角色清单)右边是你上传文本时分隔符左边设置的说话人:{ChoseSpeakerFromConfigList}|{SeakerInUploadText}",
+                                        placeholder = "ましろ|真白\n七深|七深\n透子|透子\nつくし|筑紫\n瑠唯|瑠唯\nそよ|素世\n祥子|祥子",
+                    )                  
+                    speaker = gr.Dropdown(
+                        choices=speakers, value = "ましろ", label="选择默认说话人"
+                    )
+                with gr.Column():
+                    sdp_ratio = gr.Slider(
+                    minimum=0, maximum=1, value=0.2, step=0.01, label="SDP/DP混合比"
+                    )
+                    noise_scale = gr.Slider(
+                        minimum=0.1, maximum=2, value=0.6, step=0.01, label="感情调节"
+                    )
+                    noise_scale_w = gr.Slider(
+                        minimum=0.1, maximum=2, value=0.667, step=0.01, label="音素长度"
+                    )
+                    length_scale = gr.Slider(
+                        minimum=0.1, maximum=2, value=1, step=0.01, label="生成长度"
+                    )
+                    LastAudioOutput = gr.Audio(label="当使用cuda时才能在本地文件夹浏览全部文件")
+                    btn2 = gr.Button("点击生成", variant="primary")
+                btn2.click(
+                    audiobook,
+                    inputs=[
+                        inputFile,
+                        groupSize,
+                        speaker,
+                        sdp_ratio,
+                        noise_scale,
+                        noise_scale_w,
+                        length_scale,
+                        spealerList,
+                        silenceTime,
+                        filepath,
+                        raw_text
+                    ],
+                    outputs=[LastAudioOutput],
+                )
     print("推理页面已开启!")
     app.launch(share=True)