Spaces:
Sleeping
Sleeping
| # Copyright 2020 Nagoya University (Tomoki Hayashi) | |
| # Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) | |
| # Adapted by Florian Lux 2021 | |
| import math | |
| import numpy as np | |
| import parselmouth | |
| import torch | |
| import torch.nn.functional as F | |
| from scipy.interpolate import interp1d | |
| class Parselmouth(torch.nn.Module): | |
| """ | |
| F0 estimation with Parselmouth https://parselmouth.readthedocs.io/en/stable/index.html | |
| """ | |
| def __init__(self, fs=16000, n_fft=1024, hop_length=256, f0min=40, f0max=600, use_token_averaged_f0=True, | |
| use_continuous_f0=True, use_log_f0=False, reduction_factor=1): | |
| super().__init__() | |
| self.fs = fs | |
| self.n_fft = n_fft | |
| self.hop_length = hop_length | |
| self.frame_period = 1000 * hop_length / fs | |
| self.f0min = f0min | |
| self.f0max = f0max | |
| self.use_token_averaged_f0 = use_token_averaged_f0 | |
| self.use_continuous_f0 = use_continuous_f0 | |
| self.use_log_f0 = use_log_f0 | |
| if use_token_averaged_f0: | |
| assert reduction_factor >= 1 | |
| self.reduction_factor = reduction_factor | |
| def output_size(self): | |
| return 1 | |
| def get_parameters(self): | |
| return dict(fs=self.fs, n_fft=self.n_fft, hop_length=self.hop_length, f0min=self.f0min, f0max=self.f0max, | |
| use_token_averaged_f0=self.use_token_averaged_f0, use_continuous_f0=self.use_continuous_f0, use_log_f0=self.use_log_f0, | |
| reduction_factor=self.reduction_factor) | |
| def forward(self, input_waves, input_waves_lengths=None, feats_lengths=None, durations=None, | |
| durations_lengths=None, norm_by_average=True, text=None): | |
| # F0 extraction | |
| pitch = self._calculate_f0(input_waves[0]) | |
| # Adjust length to match with the feature sequences | |
| pitch = self._adjust_num_frames(pitch, feats_lengths[0]).view(-1) | |
| pitch = self._average_by_duration(pitch, durations[0], text).view(-1) | |
| pitch_lengths = durations_lengths | |
| if norm_by_average: | |
| average = pitch[pitch != 0.0].mean() | |
| pitch = pitch / average | |
| # Return with the shape (B, T, 1) | |
| return pitch.unsqueeze(-1), pitch_lengths | |
| def _calculate_f0(self, input): | |
| x = input.cpu().numpy().astype(np.double) | |
| snd = parselmouth.Sound(values=x, sampling_frequency=self.fs) | |
| f0 = snd.to_pitch(time_step=self.hop_length / self.fs, pitch_floor=self.f0min, pitch_ceiling=self.f0max).selected_array['frequency'] | |
| if self.use_continuous_f0: | |
| f0 = self._convert_to_continuous_f0(f0) | |
| if self.use_log_f0: | |
| nonzero_idxs = np.where(f0 != 0)[0] | |
| f0[nonzero_idxs] = np.log(f0[nonzero_idxs]) | |
| return input.new_tensor(f0.reshape(-1), dtype=torch.float) | |
| def _adjust_num_frames(x, num_frames): | |
| if num_frames > len(x): | |
| # x = F.pad(x, (0, num_frames - len(x))) | |
| x = F.pad(x, (math.ceil((num_frames - len(x)) / 2), math.floor((num_frames - len(x)) / 2))) | |
| elif num_frames < len(x): | |
| x = x[:num_frames] | |
| return x | |
| def _convert_to_continuous_f0(f0: np.array): | |
| if (f0 == 0).all(): | |
| return f0 | |
| # padding start and end of f0 sequence | |
| start_f0 = f0[f0 != 0][0] | |
| end_f0 = f0[f0 != 0][-1] | |
| start_idx = np.where(f0 == start_f0)[0][0] | |
| end_idx = np.where(f0 == end_f0)[0][-1] | |
| f0[:start_idx] = start_f0 | |
| f0[end_idx:] = end_f0 | |
| # get non-zero frame index | |
| nonzero_idxs = np.where(f0 != 0)[0] | |
| # perform linear interpolation | |
| interp_fn = interp1d(nonzero_idxs, f0[nonzero_idxs]) | |
| f0 = interp_fn(np.arange(0, f0.shape[0])) | |
| return f0 | |
| def _average_by_duration(self, x, d, text=None): | |
| d_cumsum = F.pad(d.cumsum(dim=0), (1, 0)) | |
| x_avg = [ | |
| x[start:end].masked_select(x[start:end].gt(0.0)).mean(dim=0) if len(x[start:end].masked_select(x[start:end].gt(0.0))) != 0 else x.new_tensor(0.0) | |
| for start, end in zip(d_cumsum[:-1], d_cumsum[1:])] | |
| # find tokens that are not voiced and set pitch to 0 | |
| # while this makes sense, it makes it harder for the model to learn, so we leave this out now. | |
| # if text is not None: | |
| # for i, vector in enumerate(text): | |
| # if vector[get_feature_to_index_lookup()["voiced"]] == 0: | |
| # x_avg[i] = torch.tensor(0.0, device=x.device) | |
| return torch.stack(x_avg) | |