[WIP] Upload folder using huggingface_hub (multi-commit 513e361cfa57256a8e357aacb50b4a5aa4111aae72a68ff40bac250e9ec1f525)

README.md

@@ -1,12 +1,71 @@
----
-title: Tts Rvc Autopst
-emoji: 💬
-colorFrom: yellow
-colorTo: purple
-sdk: gradio
-sdk_version: 4.36.1
-app_file: app.py
-pinned: false
----
-
-An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).
+# Global Prosody Style Transfer Without Text Transcriptions
+
+This repository provides a PyTorch implementation of [AutoPST](https://arxiv.org/abs/2106.08519), which enables unsupervised global prosody conversion without text transcriptions.
+
+This is a short video that explains the main concepts of our work. If you find this work useful and use it in your research, please consider citing our paper.
+
+[![SpeechSplit](./assets/cover.png)](https://youtu.be/wow2DRuJ69c/)
+
+```
+@InProceedings{pmlr-v139-qian21b,
+  title = 	 {Global Prosody Style Transfer Without Text Transcriptions},
+  author =       {Qian, Kaizhi and Zhang, Yang and Chang, Shiyu and Xiong, Jinjun and Gan, Chuang and Cox, David and Hasegawa-Johnson, Mark},
+  booktitle = 	 {Proceedings of the 38th International Conference on Machine Learning},
+  pages = 	 {8650--8660},
+  year = 	 {2021},
+  editor = 	 {Meila, Marina and Zhang, Tong},
+  volume = 	 {139},
+  series = 	 {Proceedings of Machine Learning Research},
+  month = 	 {18--24 Jul},
+  publisher =    {PMLR},
+  url = 	 {http://proceedings.mlr.press/v139/qian21b.html}
+}
+
+```
+
+
+## Audio Demo
+
+The audio demo for AutoPST can be found [here](https://auspicious3000.github.io/AutoPST-Demo/)
+
+## Dependencies
+- Python 3.6
+- Numpy
+- Scipy
+- PyTorch == v1.6.0
+- librosa
+- pysptk
+- soundfile
+- wavenet_vocoder ```pip install wavenet_vocoder==0.1.1```
+  for more information, please refer to https://github.com/r9y9/wavenet_vocoder
+
+
+## To Run Demo
+
+Download [pre-trained models](https://drive.google.com/file/d/1ji3Bk6YGvXkPqFu1hLOAJp_SKw-vHGrp/view?usp=sharing) to ```assets```
+
+Download the same WaveNet vocoder model as in [AutoVC](https://github.com/auspicious3000/autovc) to ```assets```
+
+The fast and high-quality hifi-gan v1 (https://github.com/jik876/hifi-gan) pre-trained model is now available [here.](https://drive.google.com/file/d/1n76jHs8k1sDQ3Eh5ajXwdxuY_EZw4N9N/view?usp=sharing)
+
+Please refer to [AutoVC](https://github.com/auspicious3000/autovc) if you have any problems with the vocoder part, because they share the same vocoder scripts.
+
+Run ```demo.ipynb``` 
+
+
+## To Train
+
+Download [training data](https://drive.google.com/file/d/1H1dyA80qREKLHybqnYaqBRRsacIdFbnE/view?usp=sharing) to ```assets```.
+The provided training data is very small for code verification purpose only.
+Please use the scripts to prepare your own data for training.
+
+1. Prepare training data: ```python prepare_train_data.py```
+
+2. Train 1st Stage: ```python main_1.py```
+
+3. Train 2nd Stage: ```python main_2.py```
+
+
+## Final Words
+
+This project is part of an ongoing research. We hope this repo is useful for your research. If you need any help or have any suggestions on improving the framework, please raise an issue and we will do our best to get back to you as soon as possible.

app.py

@@ -13,11 +13,9 @@ import numpy as np
 import torch
 import torch.nn.functional as F
 from collections import OrderedDict
-from onmt_modules.misc import sequence_mask
-from model_autopst import Generator_2 as Predictor
-from hparams_autopst import hparams
-from model_sea import Generator
-from hparams_sea import hparams as sea_hparams
+from AutoPST.onmt_modules.misc import sequence_mask
+from AutoPST.model_autopst import Generator_2 as Predictor
+from AutoPST.hparams_autopst import hparams
 
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
@@ -27,7 +25,7 @@ checkpoint = torch.load(hf_hub_download(repo_id="jonathanjordan21/AutoPST", file
 P.load_state_dict(checkpoint['model'], strict=True)
 print('Loaded predictor .....................................................')
 
-dict_test = pickle.load(open('./assets/test_vctk.meta', 'rb'))
+dict_test = pickle.load(open('./AutoPST/assets/test_vctk.meta', 'rb'))
 
 spect_vc = OrderedDict()
 
@@ -66,17 +64,13 @@ import torch
 import librosa
 import pickle
 import os
-from synthesis import build_model
-from synthesis import wavegen
+from AutoPST.synthesis import build_model
+from AutoPST.synthesis import wavegen
 
 model = build_model().to(device)
 checkpoint = torch.load(hf_hub_download(repo_id="jonathanjordan21/AutoPST", filename="checkpoint_step001000000_ema.pth"), map_location=torch.device('cpu'))
 model.load_state_dict(checkpoint["state_dict"])
 
-# sea_checkpoint = torch.load(hf_hub_download(repo_id="jonathanjordan21/AutoPST", filename='sea.ckpt'), map_location=lambda storage, loc: storage)
-# gen =Generator(sea_hparams)
-# gen.load_state_dict(sea_checkpoint['model'], strict=True)
-
 # for name, sp in spect_vc.items():
 
 #     print(name)
@@ -87,164 +81,57 @@ model.load_state_dict(checkpoint["state_dict"])
 
 
 
-# def respond(
-#     message,
-#     history: list[tuple[str, str]],
-#     system_message,
-#     max_tokens,
-#     temperature,
-#     top_p,
-# ):
-#     messages = [{"role": "system", "content": system_message}]
+def respond(
+    message,
+    history: list[tuple[str, str]],
+    system_message,
+    max_tokens,
+    temperature,
+    top_p,
+):
+    messages = [{"role": "system", "content": system_message}]
 
-#     for val in history:
-#         if val[0]:
-#             messages.append({"role": "user", "content": val[0]})
-#         if val[1]:
-#             messages.append({"role": "assistant", "content": val[1]})
+    for val in history:
+        if val[0]:
+            messages.append({"role": "user", "content": val[0]})
+        if val[1]:
+            messages.append({"role": "assistant", "content": val[1]})
 
-#     messages.append({"role": "user", "content": message})
+    messages.append({"role": "user", "content": message})
 
-#     response = ""
+    response = ""
 
-#     for message in client.chat_completion(
-#         messages,
-#         max_tokens=max_tokens,
-#         stream=True,
-#         temperature=temperature,
-#         top_p=top_p,
-#     ):
-#         token = message.choices[0].delta.content
+    for message in client.chat_completion(
+        messages,
+        max_tokens=max_tokens,
+        stream=True,
+        temperature=temperature,
+        top_p=top_p,
+    ):
+        token = message.choices[0].delta.content
 
-#         response += token
-#         yield response
+        response += token
+        yield response
 
 """
 For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
 """
-# demo = gr.ChatInterface(
-#     respond,
-#     additional_inputs=[
-#         gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-#         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-#         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-#         gr.Slider(
-#             minimum=0.1,
-#             maximum=1.0,
-#             value=0.95,
-#             step=0.05,
-#             label="Top-p (nucleus sampling)",
-#         ),
-#     ],
-# )
-
-import os
-import pickle
-import numpy as np
-import soundfile as sf
-from scipy import signal
-from scipy.signal import get_window
-from librosa.filters import mel
-from numpy.random import RandomState
-
-
-def butter_highpass(cutoff, fs, order=5):
-    nyq = 0.5 * fs
-    normal_cutoff = cutoff / nyq
-    b, a = signal.butter(order, normal_cutoff, btype='high', analog=False)
-    return b, a
-    
-    
-def pySTFT(x, fft_length=1024, hop_length=256):
-    
-    x = np.pad(x, int(fft_length//2), mode='reflect')
-    
-    noverlap = fft_length - hop_length
-    shape = x.shape[:-1]+((x.shape[-1]-noverlap)//hop_length, fft_length)
-    strides = x.strides[:-1]+(hop_length*x.strides[-1], x.strides[-1])
-    result = np.lib.stride_tricks.as_strided(x, shape=shape,
-                                             strides=strides)
-    
-    fft_window = get_window('hann', fft_length, fftbins=True)
-    result = np.fft.rfft(fft_window * result, n=fft_length).T
-    
-    return np.abs(result)    
-    
-
-def create_sp(cep_real, spk_emb):
-    # cep_real, spk_emb = dict_test[uttr[0]][uttr[2]]
-    cep_real_A = torch.from_numpy(cep_real).unsqueeze(0).to(device)
-    len_real_A = torch.tensor(cep_real_A.size(1)).unsqueeze(0).to(device)
-    real_mask_A = sequence_mask(len_real_A, cep_real_A.size(1)).float()
-
-    # _, spk_emb = dict_test[uttr[1]][uttr[2]]
-    spk_emb_B = torch.from_numpy(spk_emb).unsqueeze(0).to(device)
-
-    with torch.no_grad():
-        spect_output, len_spect = P.infer_onmt(cep_real_A.transpose(2,1)[:,:14,:],
-                                                real_mask_A,
-                                                len_real_A,
-                                                spk_emb_B)
-
-    uttr_tgt = spect_output[:len_spect[0],0,:].cpu().numpy()
-    return uttr_tgt
-
-def create_mel(x):
-    mel_basis = mel(sr=16000, n_fft=1024, fmin=90, fmax=7600, n_mels=80).T
-    min_level = np.exp(-100 / 20 * np.log(10))
-    b, a = butter_highpass(30, 16000, order=5)
-    
-    mfcc_mean, mfcc_std, dctmx = pickle.load(open('assets/mfcc_stats.pkl', 'rb'))
-    spk2emb = pickle.load(open('assets/spk2emb_82.pkl', 'rb'))
-    
-    if x.shape[0] % 256 == 0:
-        x = np.concatenate((x, np.array([1e-06])), axis=0)
-    y = signal.filtfilt(b, a, x)
-    D = pySTFT(y * 0.96).T
-    D_mel = np.dot(D, mel_basis)
-    D_db = 20 * np.log10(np.maximum(min_level, D_mel))
-    
-    # mel sp
-    S = (D_db + 80) / 100 
-            
-    # mel cep
-    cc_tmp = S.dot(dctmx)
-    cc_norm = (cc_tmp - mfcc_mean) / mfcc_std
-    S = np.clip(S, 0, 1)
-    
-    # teacher code
-    # cc_torch = torch.from_numpy(cc_norm[:,0:20].astype(np.float32)).unsqueeze(0).to(device)
-    # with torch.no_grad():
-    #     codes = gen.encode(cc_torch, torch.ones_like(cc_torch[:,:,0])).squeeze(0)
-    return S, cc_norm
-
-def transcribe(audio, spk):
-    sr, y = audio
-    y = librosa.resample(y, orig_sr=sr, target_sr=16000)
-    y = y.astype(np.float32)
-    y /= np.max(np.abs(y))
-
-    spk_emb = np.zeros((82,))
-    spk_emb[int(spk)-1] = 1
-    
-    mel_sp, mel_cep = create_mel(y)
-    sp = create_sp(mel_cep, spk_emb)
-    waveform = wavegen(model, c=sp)
-    return 16000, waveform
-    
-    # return transcriber({"sampling_rate": sr, "raw": y})["text"]
-
-
-demo = gr.Interface(
-    transcribe,
-    [
-        gr.Audio(),
-        gr.Slider(1, 82, value=21, label="Count", step=1, info="Choose between 1 and 82")
+demo = gr.ChatInterface(
+    respond,
+    additional_inputs=[
+        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
+        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
+        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
+        gr.Slider(
+            minimum=0.1,
+            maximum=1.0,
+            value=0.95,
+            step=0.05,
+            label="Top-p (nucleus sampling)",
+        ),
     ],
-    "audio",
 )
 
 
-
 if __name__ == "__main__":
     demo.launch()

hparams.py

@@ -1,167 +0,0 @@
-class Map(dict):
-	"""
-    Example:
-    m = Map({'first_name': 'Eduardo'}, last_name='Pool', age=24, sports=['Soccer'])
-
-    Credits to epool:
-    https://stackoverflow.com/questions/2352181/how-to-use-a-dot-to-access-members-of-dictionary
-    """
-
-	def __init__(self, *args, **kwargs):
-		super(Map, self).__init__(*args, **kwargs)
-		for arg in args:
-			if isinstance(arg, dict):
-				for k, v in arg.items():
-					self[k] = v
-
-		if kwargs:
-			for k, v in kwargs.iteritems():
-				self[k] = v
-
-	def __getattr__(self, attr):
-		return self.get(attr)
-
-	def __setattr__(self, key, value):
-		self.__setitem__(key, value)
-
-	def __setitem__(self, key, value):
-		super(Map, self).__setitem__(key, value)
-		self.__dict__.update({key: value})
-
-	def __delattr__(self, item):
-		self.__delitem__(item)
-
-	def __delitem__(self, key):
-		super(Map, self).__delitem__(key)
-		del self.__dict__[key]
-
-
-# Default hyperparameters:
-hparams = Map({
-	'name': "wavenet_vocoder",
-
-	# Convenient model builder
-	'builder': "wavenet",
-
-	# Input type:
-	# 1. raw [-1, 1]
-	# 2. mulaw [-1, 1]
-	# 3. mulaw-quantize [0, mu]
-	# If input_type is raw or mulaw, network assumes scalar input and
-	# discretized mixture of logistic distributions output, otherwise one-hot
-	# input and softmax output are assumed.
-	# **NOTE**: if you change the one of the two parameters below, you need to
-	# re-run preprocessing before training.
-	'input_type': "raw",
-	'quantize_channels': 65536,  # 65536 or 256
-
-	# Audio:
-	'sample_rate': 16000,
-	# this is only valid for mulaw is True
-	'silence_threshold': 2,
-	'num_mels': 80,
-	'fmin': 125,
-	'fmax': 7600,
-	'fft_size': 1024,
-	# shift can be specified by either hop_size or frame_shift_ms
-	'hop_size': 256,
-	'frame_shift_ms': None,
-	'min_level_db': -100,
-	'ref_level_db': 20,
-	# whether to rescale waveform or not.
-	# Let x is an input waveform, rescaled waveform y is given by:
-	# y = x / np.abs(x).max() * rescaling_max
-	'rescaling': True,
-	'rescaling_max': 0.999,
-	# mel-spectrogram is normalized to [0, 1] for each utterance and clipping may
-	# happen depends on min_level_db and ref_level_db, causing clipping noise.
-	# If False, assertion is added to ensure no clipping happens.o0
-	'allow_clipping_in_normalization': True,
-
-	# Mixture of logistic distributions:
-	'log_scale_min': float(-32.23619130191664),
-
-	# Model:
-	# This should equal to `quantize_channels` if mu-law quantize enabled
-	# otherwise num_mixture * 3 (pi, mean, log_scale)
-	'out_channels': 10 * 3,
-	'layers': 24,
-	'stacks': 4,
-	'residual_channels': 512,
-	'gate_channels': 512,  # split into 2 gropus internally for gated activation
-	'skip_out_channels': 256,
-	'dropout': 1 - 0.95,
-	'kernel_size': 3,
-	# If True, apply weight normalization as same as DeepVoice3
-	'weight_normalization': True,
-	# Use legacy code or not. Default is True since we already provided a model
-	# based on the legacy code that can generate high-quality audio.
-	# Ref: https://github.com/r9y9/wavenet_vocoder/pull/73
-	'legacy': True,
-
-	# Local conditioning (set negative value to disable))
-	'cin_channels': 80,
-	# If True, use transposed convolutions to upsample conditional features,
-	# otherwise repeat features to adjust time resolution
-	'upsample_conditional_features': True,
-	# should np.prod(upsample_scales) == hop_size
-	'upsample_scales': [4, 4, 4, 4],
-	# Freq axis kernel size for upsampling network
-	'freq_axis_kernel_size': 3,
-
-	# Global conditioning (set negative value to disable)
-	# currently limited for speaker embedding
-	# this should only be enabled for multi-speaker dataset
-	'gin_channels': -1,  # i.e., speaker embedding dim
-	'n_speakers': -1,
-
-	# Data loader
-	'pin_memory': True,
-	'num_workers': 2,
-
-	# train/test
-	# test size can be specified as portion or num samples
-	'test_size': 0.0441,  # 50 for CMU ARCTIC single speaker
-	'test_num_samples': None,
-	'random_state': 1234,
-
-	# Loss
-
-	# Training:
-	'batch_size': 2,
-	'adam_beta1': 0.9,
-	'adam_beta2': 0.999,
-	'adam_eps': 1e-8,
-	'amsgrad': False,
-	'initial_learning_rate': 1e-3,
-	# see lrschedule.py for available lr_schedule
-	'lr_schedule': "noam_learning_rate_decay",
-	'lr_schedule_kwargs': {},  # {"anneal_rate": 0.5, "anneal_interval": 50000},
-	'nepochs': 2000,
-	'weight_decay': 0.0,
-	'clip_thresh': -1,
-	# max time steps can either be specified as sec or steps
-	# if both are None, then full audio samples are used in a batch
-	'max_time_sec': None,
-	'max_time_steps': 8000,
-	# Hold moving averaged parameters and use them for evaluation
-	'exponential_moving_average': True,
-	# averaged = decay * averaged + (1 - decay) * x
-	'ema_decay': 0.9999,
-
-	# Save
-	# per-step intervals
-	'checkpoint_interval': 10000,
-	'train_eval_interval': 10000,
-	# per-epoch interval
-	'test_eval_epoch_interval': 5,
-	'save_optimizer_state': True,
-
-	# Eval:
-})
-
-
-def hparams_debug_string():
-	values = hparams.values()
-	hp = ['  %s: %s' % (name, values[name]) for name in sorted(values)]
-	return 'Hyperparameters:\n' + '\n'.join(hp)

synthesis.py

@@ -1,66 +0,0 @@
-import torch
-from tqdm import tqdm
-import librosa
-from hparams import hparams
-from wavenet_vocoder import builder
-
-torch.set_num_threads(4)
-use_cuda = torch.cuda.is_available()
-device = torch.device("cuda" if use_cuda else "cpu")
-
-
-def build_model():
-    
-    model = getattr(builder, hparams.builder)(
-        out_channels=hparams.out_channels,
-        layers=hparams.layers,
-        stacks=hparams.stacks,
-        residual_channels=hparams.residual_channels,
-        gate_channels=hparams.gate_channels,
-        skip_out_channels=hparams.skip_out_channels,
-        cin_channels=hparams.cin_channels,
-        gin_channels=hparams.gin_channels,
-        weight_normalization=hparams.weight_normalization,
-        n_speakers=hparams.n_speakers,
-        dropout=hparams.dropout,
-        kernel_size=hparams.kernel_size,
-        upsample_conditional_features=hparams.upsample_conditional_features,
-        upsample_scales=hparams.upsample_scales,
-        freq_axis_kernel_size=hparams.freq_axis_kernel_size,
-        scalar_input=True,
-        legacy=hparams.legacy,
-    )
-    return model
-
-
-
-def wavegen(model, c=None, tqdm=tqdm):
-    """Generate waveform samples by WaveNet.
-    
-    """
-
-    model.eval()
-    model.make_generation_fast_()
-
-    Tc = c.shape[0]
-    upsample_factor = hparams.hop_size
-    # Overwrite length according to feature size
-    length = Tc * upsample_factor
-
-    # B x C x T
-    c = torch.FloatTensor(c.T).unsqueeze(0)
-
-    initial_input = torch.zeros(1, 1, 1).fill_(0.0)
-
-    # Transform data to GPU
-    initial_input = initial_input.to(device)
-    c = None if c is None else c.to(device)
-
-    with torch.no_grad():
-        y_hat = model.incremental_forward(
-            initial_input, c=c, g=None, T=length, tqdm=tqdm, softmax=True, quantize=True,
-            log_scale_min=hparams.log_scale_min)
-
-    y_hat = y_hat.view(-1).cpu().data.numpy()
-
-    return y_hat