update

modules/SynthesizeSegments.py

@@ -9,6 +9,7 @@ from modules.normalization import text_normalize
 import logging
 import json
 import random
+import copy
 
 from modules.speaker import Speaker
 
@@ -61,6 +62,9 @@ class SynthesizeSegments:
         self.batch_size = batch_size
 
     def segment_to_generate_params(self, segment: Dict[str, Any]) -> Dict[str, Any]:
+        if segment.get("params", None) is not None:
+            return segment["params"]
+
         text = segment.get("text", "")
         is_end = segment.get("is_end", False)
 
@@ -111,19 +115,15 @@ class SynthesizeSegments:
         for segment in segments:
             params = self.segment_to_generate_params(segment)
 
-            key_params = params
+            key_params = copy.copy(params)
             if isinstance(key_params.get("spk"), Speaker):
                 key_params["spk"] = str(key_params["spk"].id)
             key = json.dumps(
                 {k: v for k, v in key_params.items() if k != "text"}, sort_keys=True
             )
-            if params["spk"] == -1 or params["infer_seed"] == -1:
-                key = random.random()
-                buckets[key] = [segment]
-            else:
-                if key not in buckets:
-                    buckets[key] = []
-                buckets[key].append(segment)
+            if key not in buckets:
+                buckets[key] = []
+            buckets[key].append(segment)
 
         # Convert dictionary to list of buckets
         bucket_list = list(buckets.values())

modules/api/impl/ssml_api.py

@@ -23,7 +23,7 @@ from modules.api.Api import APIManager
 class SSMLRequest(BaseModel):
     ssml: str
     format: str = "mp3"
-    batch: bool = False
+    batch_size: int = 4
 
 
 async def synthesize_ssml(
@@ -34,7 +34,12 @@ async def synthesize_ssml(
     try:
         ssml = request.ssml
         format = request.format
-        batch = request.batch
+        batch_size = request.batch_size
+
+        if batch_size < 1:
+            raise HTTPException(
+                status_code=400, detail="Batch size must be greater than 0."
+            )
 
         if not ssml:
             raise HTTPException(status_code=400, detail="SSML content is required.")
@@ -43,8 +48,8 @@ async def synthesize_ssml(
         for seg in segments:
             seg["text"] = text_normalize(seg["text"], is_end=True)
 
-        if batch:
-            synthesize = SynthesizeSegments(16)
+        if batch_size != 1:
+            synthesize = SynthesizeSegments(batch_size)
             audio_segments = synthesize.synthesize_segments(segments)
             combined_audio = combine_audio_segments(audio_segments)
             buffer = io.BytesIO()

modules/synthesize_audio.py

@@ -1,12 +1,12 @@
+import io
 from modules.SentenceSplitter import SentenceSplitter
-from modules.normalization import text_normalize
+from modules.SynthesizeSegments import SynthesizeSegments, combine_audio_segments
 
 from modules import generate_audio as generate
 
 
-import numpy as np
-
 from modules.speaker import Speaker
+from modules.utils import audio
 
 
 def synthesize_audio(
@@ -39,20 +39,28 @@ def synthesize_audio(
     else:
         spliter = SentenceSplitter(spliter_threshold)
         sentences = spliter.parse(text)
-        sentences = [text_normalize(s) for s in sentences]
-        audio_data_batch = generate.generate_audio_batch(
-            texts=sentences,
-            temperature=temperature,
-            top_P=top_P,
-            top_K=top_K,
-            spk=spk,
-            infer_seed=infer_seed,
-            use_decoder=use_decoder,
-            prompt1=prompt1,
-            prompt2=prompt2,
-            prefix=prefix,
-        )
-        sample_rate = audio_data_batch[0][0]
-        audio_data = np.concatenate([data for _, data in audio_data_batch])
 
-        return sample_rate, audio_data
+        text_segments = [
+            {
+                "text": s,
+                "params": {
+                    "text": s,
+                    "temperature": temperature,
+                    "top_P": top_P,
+                    "top_K": top_K,
+                    "spk": spk,
+                    "infer_seed": infer_seed,
+                    "use_decoder": use_decoder,
+                    "prompt1": prompt1,
+                    "prompt2": prompt2,
+                    "prefix": prefix,
+                },
+            }
+            for s in sentences
+        ]
+        synthesizer = SynthesizeSegments(batch_size)
+        audio_segments = synthesizer.synthesize_segments(text_segments)
+
+        combined_audio = combine_audio_segments(audio_segments)
+
+        return audio.pydub_to_np(combined_audio)

modules/utils/audio.py

@@ -9,9 +9,12 @@ INT16_MAX = np.iinfo(np.int16).max
 
 
 def audio_to_int16(audio_data):
-    if audio_data.dtype == np.float32:
-        audio_data = (audio_data * INT16_MAX).astype(np.int16)
-    if audio_data.dtype == np.float16:
+    if (
+        audio_data.dtype == np.float32
+        or audio_data.dtype == np.float64
+        or audio_data.dtype == np.float128
+        or audio_data.dtype == np.float16
+    ):
         audio_data = (audio_data * INT16_MAX).astype(np.int16)
     return audio_data
 
@@ -27,6 +30,21 @@ def audiosegment_to_librosawav(audiosegment):
     return fp_arr
 
 
+def pydub_to_np(audio: AudioSegment) -> tuple[int, np.ndarray]:
+    """
+    Converts pydub audio segment into np.float32 of shape [duration_in_seconds*sample_rate, channels],
+    where each value is in range [-1.0, 1.0].
+    Returns tuple (audio_np_array, sample_rate).
+    """
+    return (
+        audio.frame_rate,
+        np.array(audio.get_array_of_samples(), dtype=np.float32).reshape(
+            (-1, audio.channels)
+        )
+        / (1 << (8 * audio.sample_width - 1)),
+    )
+
+
 def ndarray_to_segment(ndarray, frame_rate):
     buffer = BytesIO()
     sf.write(buffer, ndarray, frame_rate, format="wav")

webui.py

@@ -16,6 +16,8 @@ import logging
 
 from numpy import clip
 
+from modules.synthesize_audio import synthesize_audio
+
 logging.basicConfig(
     level=os.getenv("LOG_LEVEL", "INFO"),
     format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
@@ -145,8 +147,8 @@ def tts_generate(
         top_k = int(top_k)
 
     params = calc_spk_style(spk=spk, style=style)
-
     spk = params.get("spk", spk)
+
     infer_seed = infer_seed or params.get("seed", infer_seed)
     temperature = temperature or params.get("temperature", temperature)
     prefix = prefix or params.get("prefix", prefix)
@@ -159,37 +161,19 @@ def tts_generate(
     if not disable_normalize:
         text = text_normalize(text)
 
-    if batch_size == 1:
-        sample_rate, audio_data = generate_audio(
-            text=text,
-            temperature=temperature,
-            top_P=top_p,
-            top_K=top_k,
-            spk=spk,
-            infer_seed=infer_seed,
-            use_decoder=use_decoder,
-            prompt1=prompt1,
-            prompt2=prompt2,
-            prefix=prefix,
-        )
-    else:
-        spliter = SentenceSplitter(webui_config["spliter_threshold"])
-        sentences = spliter.parse(text)
-        sentences = [text_normalize(s) for s in sentences]
-        audio_data_batch = generate_audio_batch(
-            texts=sentences,
-            temperature=temperature,
-            top_P=top_p,
-            top_K=top_k,
-            spk=spk,
-            infer_seed=infer_seed,
-            use_decoder=use_decoder,
-            prompt1=prompt1,
-            prompt2=prompt2,
-            prefix=prefix,
-        )
-        sample_rate = audio_data_batch[0][0]
-        audio_data = np.concatenate([data for _, data in audio_data_batch])
+    sample_rate, audio_data = synthesize_audio(
+        text=text,
+        temperature=temperature,
+        top_P=top_p,
+        top_K=top_k,
+        spk=spk,
+        infer_seed=infer_seed,
+        use_decoder=use_decoder,
+        prompt1=prompt1,
+        prompt2=prompt2,
+        prefix=prefix,
+        batch_size=batch_size,
+    )
 
     audio_data = audio.audio_to_int16(audio_data)
     return sample_rate, audio_data