Upload 27 files

feature_extractor/preprocessor_config.json

@@ -0,0 +1,22 @@
+{
+  "chunk_length_s": 10,
+  "feature_extractor_type": "ClapFeatureExtractor",
+  "feature_size": 64,
+  "fft_window_size": 1024,
+  "frequency_max": 14000,
+  "frequency_min": 50,
+  "hop_length": 480,
+  "max_length_s": 10,
+  "n_fft": 1024,
+  "nb_frequency_bins": 513,
+  "nb_max_frames": 1000,
+  "nb_max_samples": 480000,
+  "padding": "repeatpad",
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "processor_class": "ClapProcessor",
+  "return_attention_mask": false,
+  "sampling_rate": 48000,
+  "top_db": null,
+  "truncation": "rand_trunc"
+}

language_model/config.json

@@ -0,0 +1,39 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2Model"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "max_new_tokens": 8,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.34.0",
+  "use_cache": true,
+  "vocab_size": 50257
+}

language_model/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e172b4b624b7419cf1f2c60b35837109eaa2e2378a89f0eddbc2a112a9fb9620
+size 497803738

model_index.json

@@ -0,0 +1,48 @@
+{
+  "_class_name": "AudioLDM2Pipeline",
+  "_diffusers_version": "0.22.0.dev0",
+  "feature_extractor": [
+    "transformers",
+    "ClapFeatureExtractor"
+  ],
+  "language_model": [
+    "transformers",
+    "GPT2Model"
+  ],
+  "projection_model": [
+    "audioldm2",
+    "AudioLDM2ProjectionModel"
+  ],
+  "scheduler": [
+    "diffusers",
+    "DDIMScheduler"
+  ],
+  "text_encoder": [
+    "transformers",
+    "ClapModel"
+  ],
+  "text_encoder_2": [
+    "transformers",
+    "VitsModel"
+  ],
+  "tokenizer": [
+    "transformers",
+    "RobertaTokenizerFast"
+  ],
+  "tokenizer_2": [
+    "transformers",
+    "VitsTokenizer"
+  ],
+  "unet": [
+    "audioldm2",
+    "AudioLDM2UNet2DConditionModel"
+  ],
+  "vae": [
+    "diffusers",
+    "AutoencoderKL"
+  ],
+  "vocoder": [
+    "transformers",
+    "SpeechT5HifiGan"
+  ]
+}

projection_model/config.json

@@ -0,0 +1,8 @@
+{
+  "_class_name": "AudioLDM2ProjectionModel",
+  "_diffusers_version": "0.22.0.dev0",
+  "langauge_model_dim": 768,
+  "max_seq_length": 310,
+  "text_encoder_1_dim": 192,
+  "text_encoder_dim": 512
+}

projection_model/diffusion_pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d045022a64d6688e657cc761a0cb98b3c2df9df7873fc25d860e2306652465ee
+size 2422924

scheduler/scheduler_config.json

@@ -0,0 +1,19 @@
+{
+  "_class_name": "DDIMScheduler",
+  "_diffusers_version": "0.22.0.dev0",
+  "beta_end": 0.0195,
+  "beta_schedule": "scaled_linear",
+  "beta_start": 0.0015,
+  "clip_sample": false,
+  "clip_sample_range": 1.0,
+  "dynamic_thresholding_ratio": 0.995,
+  "num_train_timesteps": 1000,
+  "prediction_type": "epsilon",
+  "rescale_betas_zero_snr": false,
+  "sample_max_value": 1.0,
+  "set_alpha_to_one": false,
+  "steps_offset": 1,
+  "thresholding": false,
+  "timestep_spacing": "leading",
+  "trained_betas": null
+}

text_encoder/config.json

@@ -0,0 +1,35 @@
+{
+  "architectures": [
+    "ClapModel"
+  ],
+  "audio_config": {
+    "depths": [
+      2,
+      2,
+      12,
+      2
+    ],
+    "fusion_num_hidden_layers": 2,
+    "hidden_size": 1024,
+    "model_type": "clap_audio_model",
+    "patch_embeds_hidden_size": 128,
+    "projection_hidden_size": 768
+  },
+  "hidden_size": 768,
+  "initializer_factor": 1.0,
+  "logit_scale_init_value": 14.285714285714285,
+  "model_type": "clap",
+  "num_hidden_layers": 16,
+  "projection_dim": 512,
+  "projection_hidden_act": "relu",
+  "text_config": {
+    "classifier_dropout": null,
+    "fusion_hidden_size": 768,
+    "fusion_num_hidden_layers": 2,
+    "initializer_range": 0.02,
+    "model_type": "clap_text_model",
+    "projection_hidden_size": 768
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.34.0"
+}

text_encoder/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5273abc4ce184eac0e026002b27b1e197037e2a107184a71f6ebb5afb4090d6a
+size 776445110

text_encoder_2/config.json

@@ -0,0 +1,82 @@
+{
+  "activation_dropout": 0.1,
+  "architectures": [
+    "VitsModel"
+  ],
+  "attention_dropout": 0.1,
+  "depth_separable_channels": 2,
+  "depth_separable_num_layers": 3,
+  "duration_predictor_dropout": 0.5,
+  "duration_predictor_filter_channels": 256,
+  "duration_predictor_flow_bins": 10,
+  "duration_predictor_kernel_size": 3,
+  "duration_predictor_num_flows": 4,
+  "duration_predictor_tail_bound": 5.0,
+  "ffn_dim": 768,
+  "ffn_kernel_size": 3,
+  "flow_size": 192,
+  "hidden_act": "relu",
+  "hidden_dropout": 0.1,
+  "hidden_size": 192,
+  "initializer_range": 0.02,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.1,
+  "leaky_relu_slope": 0.1,
+  "model_type": "vits",
+  "noise_scale": 0.667,
+  "noise_scale_duration": 0.8,
+  "num_attention_heads": 2,
+  "num_hidden_layers": 6,
+  "num_speakers": 1,
+  "posterior_encoder_num_wavenet_layers": 16,
+  "prior_encoder_num_flows": 4,
+  "prior_encoder_num_wavenet_layers": 4,
+  "resblock_dilation_sizes": [
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ]
+  ],
+  "resblock_kernel_sizes": [
+    3,
+    7,
+    11
+  ],
+  "sampling_rate": 16000,
+  "speaker_embedding_size": 0,
+  "speaking_rate": 1.0,
+  "spectrogram_bins": 513,
+  "torch_dtype": "float32",
+  "transformers_version": "4.34.0",
+  "upsample_initial_channel": 512,
+  "upsample_kernel_sizes": [
+    16,
+    16,
+    4,
+    4
+  ],
+  "upsample_rates": [
+    8,
+    8,
+    2,
+    2
+  ],
+  "use_bias": true,
+  "use_stochastic_duration_prediction": true,
+  "vocab_size": 183,
+  "wavenet_dilation_rate": 1,
+  "wavenet_dropout": 0.0,
+  "wavenet_kernel_size": 5,
+  "window_size": 4
+}

text_encoder_2/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c0825d996bb5e20ee89af0a022ba9b43af0afba2e033dac8abe83b3b45172b7
+size 145503282

tokenizer/added_tokens.json

@@ -0,0 +1,7 @@
+{
+  "</s>": 2,
+  "<mask>": 50264,
+  "<pad>": 1,
+  "<s>": 0,
+  "<unk>": 3
+}

tokenizer/special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer/tokenizer_config.json

@@ -0,0 +1,63 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50264": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "errors": "replace",
+  "mask_token": "<mask>",
+  "max_length": null,
+  "model_max_length": 512,
+  "pad_to_multiple_of": null,
+  "pad_token": "<pad>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "processor_class": "ClapProcessor",
+  "sep_token": "</s>",
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<unk>"
+}

tokenizer_2/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "_": 0
+}

tokenizer_2/special_tokens_map.json

@@ -0,0 +1,3 @@
+{
+  "pad_token": "_"
+}

tokenizer_2/tokenizer_config.json

@@ -0,0 +1,23 @@
+{
+  "add_blank": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "_",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "clean_up_tokenization_spaces": true,
+  "is_uroman": false,
+  "language": "eng",
+  "model_max_length": 310,
+  "normalize": true,
+  "pad_token": "_",
+  "phonemize": true,
+  "tokenizer_class": "VitsTokenizer",
+  "tokenizer_file": null
+}

tokenizer_2/vocab.json

@@ -0,0 +1,184 @@
+{
+  "_": 0,
+  ";": 1,
+  ":": 2,
+  ",": 3,
+  ".": 4,
+  "!": 5,
+  "?": 6,
+  "¡": 7,
+  "¿": 8,
+  "—": 9,
+  "…": 10,
+  "\"": 11,
+  "«": 12,
+  "»": 13,
+  "“": 14,
+  "”": 15,
+  " ": 16,
+  "A": 17,
+  "B": 18,
+  "C": 19,
+  "D": 20,
+  "E": 21,
+  "F": 22,
+  "G": 23,
+  "H": 24,
+  "I": 25,
+  "J": 26,
+  "K": 27,
+  "L": 28,
+  "M": 29,
+  "N": 30,
+  "O": 31,
+  "P": 32,
+  "Q": 33,
+  "R": 34,
+  "S": 35,
+  "T": 36,
+  "U": 37,
+  "V": 38,
+  "W": 39,
+  "X": 40,
+  "Y": 41,
+  "Z": 42,
+  "a": 43,
+  "b": 44,
+  "c": 45,
+  "d": 46,
+  "e": 47,
+  "f": 48,
+  "g": 49,
+  "h": 50,
+  "i": 51,
+  "j": 52,
+  "k": 53,
+  "l": 54,
+  "m": 55,
+  "n": 56,
+  "o": 57,
+  "p": 58,
+  "q": 59,
+  "r": 60,
+  "s": 61,
+  "t": 62,
+  "u": 63,
+  "v": 64,
+  "w": 65,
+  "x": 66,
+  "y": 67,
+  "z": 68,
+  "ɑ": 69,
+  "ɐ": 70,
+  "ɒ": 71,
+  "æ": 72,
+  "ɓ": 73,
+  "ʙ": 74,
+  "β": 75,
+  "ɔ": 76,
+  "ɕ": 77,
+  "ç": 78,
+  "ɗ": 79,
+  "ɖ": 80,
+  "ð": 81,
+  "ʤ": 82,
+  "ə": 83,
+  "ɘ": 84,
+  "ɚ": 85,
+  "ɛ": 86,
+  "ɜ": 87,
+  "ɝ": 88,
+  "ɞ": 89,
+  "ɟ": 90,
+  "ʄ": 91,
+  "ɡ": 92,
+  "ɠ": 93,
+  "ɢ": 94,
+  "ʛ": 95,
+  "ɦ": 96,
+  "ɧ": 97,
+  "ħ": 98,
+  "ɥ": 99,
+  "ʜ": 100,
+  "ɨ": 101,
+  "ɪ": 102,
+  "ʝ": 103,
+  "ɭ": 104,
+  "ɬ": 105,
+  "ɫ": 106,
+  "ɮ": 107,
+  "ʟ": 108,
+  "ɱ": 109,
+  "ɯ": 110,
+  "ɰ": 111,
+  "ŋ": 112,
+  "ɳ": 113,
+  "ɲ": 114,
+  "ɴ": 115,
+  "ø": 116,
+  "ɵ": 117,
+  "ɸ": 118,
+  "θ": 119,
+  "œ": 120,
+  "ɶ": 121,
+  "ʘ": 122,
+  "ɹ": 123,
+  "ɺ": 124,
+  "ɾ": 125,
+  "ɻ": 126,
+  "ʀ": 127,
+  "ʁ": 128,
+  "ɽ": 129,
+  "ʂ": 130,
+  "ʃ": 131,
+  "ʈ": 132,
+  "ʧ": 133,
+  "ʉ": 134,
+  "ʊ": 135,
+  "ʋ": 136,
+  "ⱱ": 137,
+  "ʌ": 138,
+  "ɣ": 139,
+  "ɤ": 140,
+  "ʍ": 141,
+  "χ": 142,
+  "ʎ": 143,
+  "ʏ": 144,
+  "ʑ": 145,
+  "ʐ": 146,
+  "ʒ": 147,
+  "ʔ": 148,
+  "ʡ": 149,
+  "ʕ": 150,
+  "ʢ": 151,
+  "ǀ": 152,
+  "ǁ": 153,
+  "ǂ": 154,
+  "ǃ": 155,
+  "ˈ": 156,
+  "ˌ": 157,
+  "ː": 158,
+  "ˑ": 159,
+  "ʼ": 160,
+  "ʴ": 161,
+  "ʰ": 162,
+  "ʱ": 163,
+  "ʲ": 164,
+  "ʷ": 165,
+  "ˠ": 166,
+  "ˤ": 167,
+  "˞": 168,
+  "↓": 169,
+  "↑": 170,
+  "→": 171,
+  "↗": 172,
+  "↘": 173,
+  "'": 176,
+  "̩": 175,
+  "ᵻ": 177,
+  "♪": 178,
+  "☎": 179,
+  "☒": 180,
+  "☝": 181,
+  "⚠": 182
+}

unet/config.json

@@ -0,0 +1,70 @@
+{
+  "_class_name": "AudioLDM2UNet2DConditionModel",
+  "_diffusers_version": "0.22.0.dev0",
+  "act_fn": "silu",
+  "attention_head_dim": 8,
+  "block_out_channels": [
+    128,
+    256,
+    384,
+    640
+  ],
+  "class_embed_type": null,
+  "class_embeddings_concat": false,
+  "conv_in_kernel": 3,
+  "conv_out_kernel": 3,
+  "cross_attention_dim": [
+    [
+      null,
+      768
+    ],
+    [
+      null,
+      768
+    ],
+    [
+      null,
+      768
+    ],
+    [
+      null,
+      768
+    ]
+  ],
+  "down_block_types": [
+    "DownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 8,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock2DCrossAttn",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_attention_heads": null,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "out_channels": 8,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_time_scale_shift": "default",
+  "sample_size": 262,
+  "time_cond_proj_dim": null,
+  "time_embedding_act_fn": null,
+  "time_embedding_dim": null,
+  "time_embedding_type": "positional",
+  "timestep_post_act": null,
+  "transformer_layers_per_block": 1,
+  "up_block_types": [
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D",
+    "UpBlock2D"
+  ],
+  "upcast_attention": false,
+  "use_linear_projection": false
+}

unet/diffusion_pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6c8adb14c35ce3550290c1d26202401033f2badbe6234a3404894501f27f27e
+size 1048171066

vae/config.json

@@ -0,0 +1,28 @@
+{
+  "_class_name": "AutoencoderKL",
+  "_diffusers_version": "0.22.0.dev0",
+  "act_fn": "silu",
+  "block_out_channels": [
+    128,
+    256,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "force_upcast": true,
+  "in_channels": 1,
+  "latent_channels": 8,
+  "layers_per_block": 2,
+  "norm_num_groups": 32,
+  "out_channels": 1,
+  "sample_size": 1048,
+  "scaling_factor": 0.41837412118911743,
+  "up_block_types": [
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D"
+  ]
+}

vae/diffusion_pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c4201e7ebec40dc21ed45d99077db20fd5020dbe5886269e3b83108cd2c4e16
+size 221587226

vocoder/config.json

@@ -0,0 +1,50 @@
+{
+  "architectures": [
+    "SpeechT5HifiGan"
+  ],
+  "initializer_range": 0.01,
+  "leaky_relu_slope": 0.1,
+  "model_in_dim": 64,
+  "model_type": "hifigan",
+  "normalize_before": false,
+  "resblock_dilation_sizes": [
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ]
+  ],
+  "resblock_kernel_sizes": [
+    3,
+    7,
+    11
+  ],
+  "sampling_rate": 16000,
+  "torch_dtype": "float32",
+  "transformers_version": "4.34.0",
+  "upsample_initial_channel": 1024,
+  "upsample_kernel_sizes": [
+    16,
+    16,
+    8,
+    4,
+    4
+  ],
+  "upsample_rates": [
+    5,
+    4,
+    2,
+    2,
+    2
+  ]
+}

vocoder/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bfa544480458b5175cb0664a2bc1cc615c2b83c19c71d2c8c3002bba3c93bde
+size 221120794