Upload 6 files

utils/README.md

@@ -0,0 +1,6 @@
+# External Colab Code
+Code used to make Google Colab work correctly
+- Repo link: https://github.com/IAHispano/Applio-RVC-Fork/
+
+Thanks to https://github.com/kalomaze/externalcolabcode
+

utils/backups.py

@@ -0,0 +1,141 @@
+import os
+import shutil
+import hashlib
+import time
+import base64
+
+
+
+
+LOGS_FOLDER = '/content/Applio-RVC-Fork/logs'
+WEIGHTS_FOLDER = '/content/Applio-RVC-Fork/weights'
+GOOGLE_DRIVE_PATH = '/content/drive/MyDrive/RVC_Backup'
+
+def import_google_drive_backup():
+    print("Importing Google Drive backup...")
+    weights_exist = False
+    for root, dirs, files in os.walk(GOOGLE_DRIVE_PATH):
+        for filename in files:
+            filepath = os.path.join(root, filename)
+            if os.path.isfile(filepath) and not filepath.startswith(os.path.join(GOOGLE_DRIVE_PATH, 'weights')):
+                backup_filepath = os.path.join(LOGS_FOLDER, os.path.relpath(filepath, GOOGLE_DRIVE_PATH))
+                backup_folderpath = os.path.dirname(backup_filepath)
+                if not os.path.exists(backup_folderpath):
+                    os.makedirs(backup_folderpath)
+                    print(f'Created backup folder: {backup_folderpath}', flush=True)
+                shutil.copy2(filepath, backup_filepath) # copy file with metadata
+                print(f'Imported file from Google Drive backup: {filename}')
+            elif filepath.startswith(os.path.join(GOOGLE_DRIVE_PATH, 'weights')) and filename.endswith('.pth'):
+                weights_exist = True
+                weights_filepath = os.path.join(WEIGHTS_FOLDER, os.path.relpath(filepath, os.path.join(GOOGLE_DRIVE_PATH, 'weights')))
+                weights_folderpath = os.path.dirname(weights_filepath)
+                if not os.path.exists(weights_folderpath):
+                    os.makedirs(weights_folderpath)
+                    print(f'Created weights folder: {weights_folderpath}', flush=True)
+                shutil.copy2(filepath, weights_filepath) # copy file with metadata
+                print(f'Imported file from weights: {filename}')
+    if weights_exist:
+        print("Copied weights from Google Drive backup to local weights folder.")
+    else:
+        print("No weights found in Google Drive backup.")
+    print("Google Drive backup import completed.")
+
+def get_md5_hash(file_path):
+    hash_md5 = hashlib.md5()
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+def copy_weights_folder_to_drive():
+    destination_folder = os.path.join(GOOGLE_DRIVE_PATH, 'weights')
+    try:
+        if not os.path.exists(destination_folder):
+            os.makedirs(destination_folder)
+
+        num_copied = 0
+        for filename in os.listdir(WEIGHTS_FOLDER):
+            if filename.endswith('.pth'):
+                source_file = os.path.join(WEIGHTS_FOLDER, filename)
+                destination_file = os.path.join(destination_folder, filename)
+                if not os.path.exists(destination_file):
+                    shutil.copy2(source_file, destination_file)
+                    num_copied += 1
+                    print(f"Copied {filename} to Google Drive!")
+
+        if num_copied == 0:
+            print("No new finished models found for copying.")
+        else:
+            print(f"Finished copying {num_copied} files to Google Drive!")
+
+    except Exception as e:
+        print(f"An error occurred while copying weights: {str(e)}")
+        # You can log the error or take appropriate actions here.
+
+def backup_files():
+    print("\nStarting backup loop...")
+    last_backup_timestamps_path = os.path.join(LOGS_FOLDER, 'last_backup_timestamps.txt')
+    fully_updated = False  # boolean to track if all files are up to date
+    
+    while True:
+        try:
+            updated = False  # flag to check if any files were updated
+            last_backup_timestamps = {}
+
+            try:
+                with open(last_backup_timestamps_path, 'r') as f:
+                    last_backup_timestamps = dict(line.strip().split(':') for line in f)
+            except FileNotFoundError:
+                pass  # File does not exist yet, which is fine
+            
+            for root, dirs, files in os.walk(LOGS_FOLDER):
+                for filename in files:
+                    if filename != 'last_backup_timestamps.txt':
+                        filepath = os.path.join(root, filename)
+                        if os.path.isfile(filepath):
+                            backup_filepath = os.path.join(GOOGLE_DRIVE_PATH, os.path.relpath(filepath, LOGS_FOLDER))
+                            backup_folderpath = os.path.dirname(backup_filepath)
+                            if not os.path.exists(backup_folderpath):
+                                os.makedirs(backup_folderpath)
+                                print(f'Created backup folder: {backup_folderpath}', flush=True)
+                            # check if file has changed since last backup
+                            last_backup_timestamp = last_backup_timestamps.get(filepath)
+                            current_timestamp = os.path.getmtime(filepath)
+                            if last_backup_timestamp is None or float(last_backup_timestamp) < current_timestamp:
+                                shutil.copy2(filepath, backup_filepath)  # copy file with metadata
+                                last_backup_timestamps[filepath] = str(current_timestamp)  # update last backup timestamp
+                                if last_backup_timestamp is None:
+                                    print(f'Backed up file: {filename}')
+                                else:
+                                    print(f'Updating backed up file: {filename}')
+                                updated = True
+                                fully_updated = False  # if a file is updated, all files are not up to date
+            
+            # check if any files were deleted in Colab and delete them from the backup drive
+            for filepath in list(last_backup_timestamps.keys()):
+                if not os.path.exists(filepath):
+                    backup_filepath = os.path.join(GOOGLE_DRIVE_PATH, os.path.relpath(filepath, LOGS_FOLDER))
+                    if os.path.exists(backup_filepath):
+                        os.remove(backup_filepath)
+                        print(f'Deleted file: {filepath}')
+                    del last_backup_timestamps[filepath]
+                    updated = True
+                    fully_updated = False  # if a file is deleted, all files are not up to date
+            
+            if not updated and not fully_updated:
+                print("Files are up to date.")
+                fully_updated = True  # if all files are up to date, set the boolean to True
+                copy_weights_folder_to_drive()
+                sleep_time = 15
+            else:
+                sleep_time = 0.1
+            
+            with open(last_backup_timestamps_path, 'w') as f:
+                for filepath, timestamp in last_backup_timestamps.items():
+                    f.write(f'{filepath}:{timestamp}\n')
+            
+            time.sleep(sleep_time)  # wait for 15 seconds before checking again, or 0.1s if not fully up to date to speed up backups
+        
+        except Exception as e:
+            print(f"An error occurred: {str(e)}")
+            # You can log the error or take appropriate actions here.

utils/backups_test.py

@@ -0,0 +1,138 @@
+
+import os
+import shutil
+import hashlib
+import time
+
+LOGS_FOLDER = '/content/Applio-RVC-Fork/logs'
+WEIGHTS_FOLDER = '/content/Applio-RVC-Fork/weights'
+GOOGLE_DRIVE_PATH = '/content/drive/MyDrive/RVC_Backup'
+
+def import_google_drive_backup():
+    print("Importing Google Drive backup...")
+    GOOGLE_DRIVE_PATH = '/content/drive/MyDrive/RVC_Backup'  # change this to your Google Drive path
+    LOGS_FOLDER = '/content/Applio-RVC-Fork/logs'
+    WEIGHTS_FOLDER = '/content/Applio-RVC-Fork/weights'
+    weights_exist = False
+    files_to_copy = []
+    weights_to_copy = []
+    
+    def handle_files(root, files, is_weight_files=False):
+        for filename in files:
+            filepath = os.path.join(root, filename)
+            if filename.endswith('.pth') and is_weight_files:
+                weights_exist = True
+                backup_filepath = os.path.join(WEIGHTS_FOLDER, os.path.relpath(filepath, GOOGLE_DRIVE_PATH))
+            else:
+                backup_filepath = os.path.join(LOGS_FOLDER, os.path.relpath(filepath, GOOGLE_DRIVE_PATH))
+            backup_folderpath = os.path.dirname(backup_filepath)
+            if not os.path.exists(backup_folderpath):
+                os.makedirs(backup_folderpath)
+                print(f'Created folder: {backup_folderpath}', flush=True)
+            if is_weight_files:
+                weights_to_copy.append((filepath, backup_filepath))
+            else:
+                files_to_copy.append((filepath, backup_filepath))
+
+    for root, dirs, files in os.walk(os.path.join(GOOGLE_DRIVE_PATH, 'logs')):
+        handle_files(root, files)
+    
+    for root, dirs, files in os.walk(os.path.join(GOOGLE_DRIVE_PATH, 'weights')):
+        handle_files(root, files, True)
+
+    # Copy files in batches
+    total_files = len(files_to_copy)
+    start_time = time.time()
+    for i, (source, dest) in enumerate(files_to_copy, start=1):
+        with open(source, 'rb') as src, open(dest, 'wb') as dst:
+            shutil.copyfileobj(src, dst, 1024*1024)  # 1MB buffer size
+        # Report progress every 5 seconds or after every 100 files, whichever is less frequent
+        if time.time() - start_time > 5 or i % 100 == 0:
+            print(f'\rCopying file {i} of {total_files} ({i * 100 / total_files:.2f}%)', end="")
+            start_time = time.time()
+    print(f'\nImported {len(files_to_copy)} files from Google Drive backup')
+
+    # Copy weights in batches
+    total_weights = len(weights_to_copy)
+    start_time = time.time()
+    for i, (source, dest) in enumerate(weights_to_copy, start=1):
+        with open(source, 'rb') as src, open(dest, 'wb') as dst:
+            shutil.copyfileobj(src, dst, 1024*1024)  # 1MB buffer size
+        # Report progress every 5 seconds or after every 100 files, whichever is less frequent
+        if time.time() - start_time > 5 or i % 100 == 0:
+            print(f'\rCopying weight file {i} of {total_weights} ({i * 100 / total_weights:.2f}%)', end="")
+            start_time = time.time()
+    if weights_exist:
+        print(f'\nImported {len(weights_to_copy)} weight files')
+        print("Copied weights from Google Drive backup to local weights folder.")
+    else:
+        print("\nNo weights found in Google Drive backup.")
+    print("Google Drive backup import completed.")
+
+def backup_files():
+    print("\n Starting backup loop...")
+    last_backup_timestamps_path = os.path.join(LOGS_FOLDER, 'last_backup_timestamps.txt')
+    fully_updated = False  # boolean to track if all files are up to date
+    try:
+        with open(last_backup_timestamps_path, 'r') as f:
+            last_backup_timestamps = dict(line.strip().split(':') for line in f)
+    except:
+        last_backup_timestamps = {}
+
+    while True:
+        updated = False
+        files_to_copy = []
+        files_to_delete = []
+
+        for root, dirs, files in os.walk(LOGS_FOLDER):
+            for filename in files:
+                if filename != 'last_backup_timestamps.txt':
+                    filepath = os.path.join(root, filename)
+                    if os.path.isfile(filepath):
+                        backup_filepath = os.path.join(GOOGLE_DRIVE_PATH, os.path.relpath(filepath, LOGS_FOLDER))
+                        backup_folderpath = os.path.dirname(backup_filepath)
+
+                        if not os.path.exists(backup_folderpath):
+                            os.makedirs(backup_folderpath)
+                            print(f'Created backup folder: {backup_folderpath}', flush=True)
+
+                        # check if file has changed since last backup
+                        last_backup_timestamp = last_backup_timestamps.get(filepath)
+                        current_timestamp = os.path.getmtime(filepath)
+                        if last_backup_timestamp is None or float(last_backup_timestamp) < current_timestamp:
+                            files_to_copy.append((filepath, backup_filepath))  # add to list of files to copy
+                            last_backup_timestamps[filepath] = str(current_timestamp)  # update last backup timestamp
+                            updated = True
+                            fully_updated = False  # if a file is updated, all files are not up to date
+
+        # check if any files were deleted in Colab and delete them from the backup drive
+        for filepath in list(last_backup_timestamps.keys()):
+            if not os.path.exists(filepath):
+                backup_filepath = os.path.join(GOOGLE_DRIVE_PATH, os.path.relpath(filepath, LOGS_FOLDER))
+                if os.path.exists(backup_filepath):
+                    files_to_delete.append(backup_filepath)  # add to list of files to delete
+                del last_backup_timestamps[filepath]
+                updated = True
+                fully_updated = False  # if a file is deleted, all files are not up to date
+
+        # Copy files in batches
+        if files_to_copy:
+            for source, dest in files_to_copy:
+                shutil.copy2(source, dest)
+            print(f'Copied or updated {len(files_to_copy)} files')
+
+        # Delete files in batches
+        if files_to_delete:
+            for file in files_to_delete:
+                os.remove(file)
+            print(f'Deleted {len(files_to_delete)} files')
+
+        if not updated and not fully_updated:
+            print("Files are up to date.")
+            fully_updated = True  # if all files are up to date, set the boolean to True
+            copy_weights_folder_to_drive()
+
+        with open(last_backup_timestamps_path, 'w') as f:
+            for filepath, timestamp in last_backup_timestamps.items():
+                f.write(f'{filepath}:{timestamp}\n')
+        time.sleep(15)  # wait for 15 seconds before checking again

utils/clonerepo_experimental.py

@@ -0,0 +1,253 @@
+import os
+import subprocess
+import shutil
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm.notebook import tqdm
+from pathlib import Path
+import requests
+
+def run_script():
+    def run_cmd(cmd):
+        process = subprocess.run(cmd, shell=True, check=True, text=True)
+        return process.stdout
+
+    # Change the current directory to /content/
+    os.chdir('/content/')
+    print("Changing dir to /content/")
+
+    # Your function to edit the file
+    def edit_file(file_path):
+        temp_file_path = "/tmp/temp_file.py"
+        changes_made = False
+        with open(file_path, "r") as file, open(temp_file_path, "w") as temp_file:
+            previous_line = ""
+            second_previous_line = ""
+            for line in file:
+                new_line = line.replace("value=160", "value=128")
+                if new_line != line:
+                    print("Replaced 'value=160' with 'value=128'")
+                    changes_made = True
+                line = new_line
+
+                new_line = line.replace("crepe hop length: 160", "crepe hop length: 128")
+                if new_line != line:
+                    print("Replaced 'crepe hop length: 160' with 'crepe hop length: 128'")
+                    changes_made = True
+                line = new_line
+
+                new_line = line.replace("value=0.88", "value=0.75")
+                if new_line != line:
+                    print("Replaced 'value=0.88' with 'value=0.75'")
+                    changes_made = True
+                line = new_line
+
+                if "label=i18n(\"输入源音量包络替换输出音量包络融合比例，越靠近1越使用输出包络\")" in previous_line and "value=1," in line:
+                    new_line = line.replace("value=1,", "value=0.25,")
+                    if new_line != line:
+                        print("Replaced 'value=1,' with 'value=0.25,' based on the condition")
+                        changes_made = True
+                    line = new_line
+
+                if "label=i18n(\"总训练轮数total_epoch\")" in previous_line and "value=20," in line:
+                    new_line = line.replace("value=20,", "value=500,")
+                    if new_line != line:
+                        print("Replaced 'value=20,' with 'value=500,' based on the condition for DEFAULT EPOCH")
+                        changes_made = True
+                    line = new_line
+
+                if 'choices=["pm", "harvest", "dio", "crepe", "crepe-tiny", "mangio-crepe", "mangio-crepe-tiny"], # Fork Feature. Add Crepe-Tiny' in previous_line:
+                    if 'value="pm",' in line:
+                        new_line = line.replace('value="pm",', 'value="mangio-crepe",')
+                        if new_line != line:
+                            print("Replaced 'value=\"pm\",' with 'value=\"mangio-crepe\",' based on the condition")
+                            changes_made = True
+                        line = new_line
+
+                new_line = line.replace('label=i18n("输入训练文件夹路径"), value="E:\\\\语音音频+标注\\\\米津玄师\\\\src"', 'label=i18n("输入训练文件夹路径"), value="/content/dataset/"')
+                if new_line != line:
+                    print("Replaced 'label=i18n(\"输入训练文件夹路径\"), value=\"E:\\\\语音音频+标注\\\\米津玄师\\\\src\"' with 'label=i18n(\"输入训练文件夹路径\"), value=\"/content/dataset/\"'")
+                    changes_made = True
+                line = new_line
+
+                if 'label=i18n("是否仅保存最新的ckpt文件以节省硬盘空间"),' in second_previous_line:
+                    if 'value=i18n("否"),' in line:
+                        new_line = line.replace('value=i18n("否"),', 'value=i18n("是"),')
+                        if new_line != line:
+                            print("Replaced 'value=i18n(\"否\"),' with 'value=i18n(\"是\"),' based on the condition for SAVE ONLY LATEST")
+                            changes_made = True
+                        line = new_line
+
+                if 'label=i18n("是否在每次保存时间点将最终小模型保存至weights文件夹"),' in second_previous_line:
+                    if 'value=i18n("否"),' in line:
+                        new_line = line.replace('value=i18n("否"),', 'value=i18n("是"),')
+                        if new_line != line:
+                            print("Replaced 'value=i18n(\"否\"),' with 'value=i18n(\"是\"),' based on the condition for SAVE SMALL WEIGHTS")
+                            changes_made = True
+                        line = new_line
+
+                temp_file.write(line)
+                second_previous_line = previous_line
+                previous_line = line
+
+        # After finished, we replace the original file with the temp one
+        import shutil
+        shutil.move(temp_file_path, file_path)
+
+        if changes_made:
+            print("Changes made and file saved successfully.")
+        else:
+            print("No changes were needed.")
+
+    # Define the repo path
+    repo_path = '/content/Applio-RVC-Fork'
+
+    def copy_all_files_in_directory(src_dir, dest_dir):
+        # Iterate over all files in source directory
+        for item in Path(src_dir).glob('*'):
+            if item.is_file():
+                # Copy each file to destination directory
+                shutil.copy(item, dest_dir)
+            else:
+                # If it's a directory, make a new directory in the destination and copy the files recursively
+                new_dest = Path(dest_dir) / item.name
+                new_dest.mkdir(exist_ok=True)
+                copy_all_files_in_directory(str(item), str(new_dest))
+
+    def clone_and_copy_repo(repo_path):
+        # New repository link
+        new_repo_link = "https://github.com/IAHispano/Applio-RVC-Fork/"
+        # Temporary path to clone the repository
+        temp_repo_path = "/content/temp_Applio-RVC-Fork"
+        # New folder name
+        new_folder_name = "Applio-RVC-Fork"
+
+        # Clone the latest code from the new repository to a temporary location
+        run_cmd(f"git clone {new_repo_link} {temp_repo_path}")
+        os.chdir(temp_repo_path)
+
+        run_cmd(f"git checkout 3fa4dad3d8961e5ca2522e9e12c0b4ddb71ad402")
+        run_cmd(f"git checkout f9e606c279cb49420597519b0a83b92be81e42e4")
+        run_cmd(f"git checkout 9e305588844c5442d58add1061b29beeca89d679")
+        run_cmd(f"git checkout bf92dc1eb54b4f28d6396a4d1820a25896cc9af8")
+        run_cmd(f"git checkout c3810e197d3cb98039973b2f723edf967ecd9e61")
+        run_cmd(f"git checkout a33159efd134c2413b0afe26a76b7dc87926d2de")
+        run_cmd(f"git checkout 24e251fb62c662e39ac5cf9253cc65deb9be94ec")
+        run_cmd(f"git checkout ad5667d3017e93232dba85969cddac1322ba2902")
+        run_cmd(f"git checkout ce9715392cf52dd5a0e18e00d1b5e408f08dbf27")
+        run_cmd(f"git checkout 7c7da3f2ac68f3bd8f3ad5ca5c700f18ab9f90eb")
+        run_cmd(f"git checkout 4ac395eab101955e8960b50d772c26f592161764")
+        run_cmd(f"git checkout b15b358702294c7375761584e5276c811ffab5e8")
+        run_cmd(f"git checkout 1501793dc490982db9aca84a50647764caa66e51")
+        run_cmd(f"git checkout 21f7faf57219c75e6ba837062350391a803e9ae2")
+        run_cmd(f"git checkout b5eb689fbc409b49f065a431817f822f554cebe7")
+        run_cmd(f"git checkout 7e02fae1ebf24cb151bf6cbe787d06734aa65862")
+        run_cmd(f"git checkout 6aea5ea18ed0b9a1e03fa5d268d6bc3c616672a9")
+        run_cmd(f"git checkout f0f9b25717e59116473fb42bd7f9252cfc32b398")
+        run_cmd(f"git checkout b394de424088a81fc081224bc27338a8651ad3b2")
+        run_cmd(f"git checkout f1999406a88b80c965d2082340f5ea2bfa9ab67a")
+        run_cmd(f"git checkout d98a0fa8dc715308dfc73eac5c553b69c6ee072b")
+        run_cmd(f"git checkout d73267a415fb0eba98477afa43ef71ffd82a7157")
+        run_cmd(f"git checkout 1a03d01356ae79179e1fb8d8915dc9cc79925742")
+        run_cmd(f"git checkout 81497bb3115e92c754300c9b3992df428886a3e9")
+        run_cmd(f"git checkout c5af1f8edcf79cb70f065c0110e279e78e48caf9")
+        run_cmd(f"git checkout cdb3c90109387fa4dfa92f53c3864c71170ffc77")
+
+        # Edit the file here, before copying
+        #edit_file(f"{temp_repo_path}/infer-web.py")
+
+        # Copy all files from the cloned repository to the existing path
+        copy_all_files_in_directory(temp_repo_path, repo_path)
+        print(f"Copying all {new_folder_name} files from GitHub.")
+
+        # Change working directory back to /content/
+        os.chdir('/content/')
+        print("Changed path back to /content/")
+        
+        # Remove the temporary cloned repository
+        shutil.rmtree(temp_repo_path)
+
+    # Call the function
+    clone_and_copy_repo(repo_path)
+
+    # Download the credentials file for RVC archive sheet
+    os.makedirs('/content/Applio-RVC-Fork/stats/', exist_ok=True)
+    run_cmd("wget -q https://cdn.discordapp.com/attachments/945486970883285045/1114717554481569802/peppy-generator-388800-07722f17a188.json -O /content/Applio-RVC-Fork/stats/peppy-generator-388800-07722f17a188.json")
+
+    # Forcefully delete any existing torchcrepe dependencies downloaded from an earlier run just in case
+    shutil.rmtree('/content/Applio-RVC-Fork/torchcrepe', ignore_errors=True)
+    shutil.rmtree('/content/torchcrepe', ignore_errors=True)
+
+    # Download the torchcrepe folder from the maxrmorrison/torchcrepe repository
+    run_cmd("git clone https://github.com/maxrmorrison/torchcrepe.git")
+    shutil.move('/content/torchcrepe/torchcrepe', '/content/Applio-RVC-Fork/')
+    shutil.rmtree('/content/torchcrepe', ignore_errors=True)  # Delete the torchcrepe repository folder
+
+    # Change the current directory to /content/Applio-RVC-Fork
+    os.chdir('/content/Applio-RVC-Fork')
+    os.makedirs('pretrained', exist_ok=True)
+    os.makedirs('uvr5_weights', exist_ok=True)
+
+def download_file(url, filepath):
+    response = requests.get(url, stream=True)
+    response.raise_for_status()
+
+    with open(filepath, "wb") as file:
+        for chunk in response.iter_content(chunk_size=8192):
+            if chunk:
+                file.write(chunk)
+
+def download_pretrained_models():
+    pretrained_models = {
+        "pretrained": [
+            "D40k.pth",
+            "G40k.pth",
+            "f0D40k.pth",
+            "f0G40k.pth"
+        ],
+        "pretrained_v2": [
+            "D40k.pth",
+            "G40k.pth",
+            "f0D40k.pth",
+            "f0G40k.pth",
+            "f0G48k.pth",
+            "f0D48k.pth"
+        ],
+        "uvr5_weights": [
+            "HP2-人声vocals+非人声instrumentals.pth",
+            "HP5-主旋律人声vocals+其他instrumentals.pth",
+            "VR-DeEchoNormal.pth",
+            "VR-DeEchoDeReverb.pth",
+            "VR-DeEchoAggressive.pth",
+            "HP5_only_main_vocal.pth",
+            "HP3_all_vocals.pth",
+            "HP2_all_vocals.pth"
+        ]
+    }
+    part2 = "I"
+    base_url = "https://huggingface.co/lj1995/VoiceConversionWebU" + part2 + "/resolve/main/"
+    base_path = "/content/Applio-RVC-Fork/"
+    base_pathm = base_path
+
+    # Calculate total number of files to download
+    total_files = sum(len(files) for files in pretrained_models.values()) + 1  # +1 for hubert_base.pt
+
+    with tqdm(total=total_files, desc="Downloading files") as pbar:
+        for folder, models in pretrained_models.items():
+            folder_path = os.path.join(base_path, folder)
+            os.makedirs(folder_path, exist_ok=True)
+            for model in models:
+                url = base_url + folder + "/" + model
+                filepath = os.path.join(folder_path, model)
+                download_file(url, filepath)
+                pbar.update()
+
+        # Download hubert_base.pt to the base path
+        hubert_url = base_url + "hubert_base.pt"
+        hubert_filepath = os.path.join(base_pathm, "hubert_base.pt")
+        download_file(hubert_url, hubert_filepath)
+        pbar.update()
+def clone_repository(run_download):
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        executor.submit(run_script)
+        if run_download:
+            executor.submit(download_pretrained_models)

utils/dependency.py

@@ -0,0 +1,170 @@
+import os
+import csv
+import shutil
+import tarfile
+import subprocess
+from pathlib import Path
+from datetime import datetime
+
+def install_packages_but_jank_af():
+    packages = ['build-essential', 'python3-dev', 'ffmpeg', 'aria2']
+    pip_packages = ['pip', 'setuptools', 'wheel', 'httpx==0.23.0', 'faiss-gpu', 'fairseq', 'gradio==3.34.0',
+                    'ffmpeg', 'ffmpeg-python', 'praat-parselmouth', 'pyworld', 'numpy==1.23.5',
+                    'numba==0.56.4', 'librosa==0.9.2', 'mega.py', 'gdown', 'onnxruntime', 'pyngrok==4.1.12',
+                    'gTTS', 'elevenlabs', 'wget', 'tensorboardX', 'unidecode', 'huggingface-hub', 'stftpitchshift==1.5.1', 
+                    'yt-dlp', 'pedalboard', 'pathvalidate', 'nltk', 'edge-tts', 'git+https://github.com/suno-ai/bark.git', 'python-dotenv' , 'av']
+
+    print("Updating and installing system packages...")
+    for package in packages:
+        print(f"Installing {package}...")
+        subprocess.check_call(['apt-get', 'install', '-qq', '-y', package])
+
+    print("Updating and installing pip packages...")
+    subprocess.check_call(['pip', 'install', '--upgrade'] + pip_packages)
+
+    print('Packages up to date.')
+
+
+def setup_environment(ForceUpdateDependencies, ForceTemporaryStorage):
+    # Mounting Google Drive
+    if not ForceTemporaryStorage:
+        from google.colab import drive
+
+        if not os.path.exists('/content/drive'):
+            drive.mount('/content/drive')
+        else:
+            print('Drive is already mounted. Proceeding...')
+
+    # Function to install dependencies with progress
+    def install_packages():
+        packages = ['build-essential', 'python3-dev', 'ffmpeg', 'aria2']
+        pip_packages = ['pip', 'setuptools', 'wheel', 'httpx==0.23.0', 'faiss-gpu', 'fairseq', 'gradio==3.34.0',
+                        'ffmpeg', 'ffmpeg-python', 'praat-parselmouth', 'pyworld', 'numpy==1.23.5',
+                        'numba==0.56.4', 'librosa==0.9.2', 'mega.py', 'gdown', 'onnxruntime', 'pyngrok==4.1.12',
+                        'gTTS', 'elevenlabs', 'wget', 'tensorboardX', 'unidecode', 'huggingface-hub', 'stftpitchshift==1.5.1', 
+                        'yt-dlp', 'pedalboard', 'pathvalidate', 'nltk', 'edge-tts', 'git+https://github.com/suno-ai/bark.git', 'python-dotenv' , 'av']
+
+        print("Updating and installing system packages...")
+        for package in packages:
+            print(f"Installing {package}...")
+            subprocess.check_call(['apt-get', 'install', '-qq', '-y', package])
+
+        print("Updating and installing pip packages...")
+        subprocess.check_call(['pip', 'install', '--upgrade'] + pip_packages)
+
+
+        print('Packages up to date.')
+
+    # Function to scan a directory and writes filenames and timestamps
+    def scan_and_write(base_path, output_file):
+        with open(output_file, 'w', newline='') as f:
+            writer = csv.writer(f)
+            for dirpath, dirs, files in os.walk(base_path):
+                for filename in files:
+                    fname = os.path.join(dirpath, filename)
+                    try:
+                        mtime = os.path.getmtime(fname)
+                        writer.writerow([fname, mtime])
+                    except Exception as e:
+                        print(f'Skipping irrelevant nonexistent file {fname}: {str(e)}')
+        print(f'Finished recording filesystem timestamps to {output_file}.')
+
+    # Function to compare files
+    def compare_files(old_file, new_file):
+        old_files = {}
+        new_files = {}
+
+        with open(old_file, 'r') as f:
+            reader = csv.reader(f)
+            old_files = {rows[0]:rows[1] for rows in reader}
+
+        with open(new_file, 'r') as f:
+            reader = csv.reader(f)
+            new_files = {rows[0]:rows[1] for rows in reader}
+
+        removed_files = old_files.keys() - new_files.keys()
+        added_files = new_files.keys() - old_files.keys()
+        unchanged_files = old_files.keys() & new_files.keys()
+
+        changed_files = {f for f in unchanged_files if old_files[f] != new_files[f]}
+
+        for file in removed_files:
+            print(f'File has been removed: {file}')
+
+        for file in changed_files:
+            print(f'File has been updated: {file}')
+
+        return list(added_files) + list(changed_files)
+
+    # Check if CachedRVC.tar.gz exists
+    if ForceTemporaryStorage:
+        file_path = '/content/CachedRVC.tar.gz'
+    else:
+        file_path = '/content/drive/MyDrive/RVC_Cached/CachedRVC.tar.gz'
+
+    content_file_path = '/content/CachedRVC.tar.gz'
+    extract_path = '/'
+
+    if not os.path.exists(file_path):
+        folder_path = os.path.dirname(file_path)
+        os.makedirs(folder_path, exist_ok=True)
+        print('No cached dependency install found. Attempting to download GitHub backup..')
+
+        try:
+            download_url = "https://github.com/kalomaze/QuickMangioFixes/releases/download/release3/CachedRVC.tar.gz"
+            subprocess.run(["wget", "-O", file_path, download_url])
+            print('Download completed successfully!')
+        except Exception as e:
+            print('Download failed:', str(e))
+
+            # Delete the failed download file
+            if os.path.exists(file_path):
+                os.remove(file_path)
+            print('Failed download file deleted. Continuing manual backup..')
+
+    if Path(file_path).exists():
+        if ForceTemporaryStorage:
+            print('Finished downloading CachedRVC.tar.gz.')
+        else:
+            print('CachedRVC.tar.gz found on Google Drive. Proceeding to copy and extract...')
+
+        # Check if ForceTemporaryStorage is True and skip copying if it is
+        if ForceTemporaryStorage:
+            pass
+        else:
+            shutil.copy(file_path, content_file_path)
+
+        print('Beginning backup copy operation...')
+
+        with tarfile.open(content_file_path, 'r:gz') as tar:
+            for member in tar.getmembers():
+                target_path = os.path.join(extract_path, member.name)
+                try:
+                    tar.extract(member, extract_path)
+                except Exception as e:
+                    print('Failed to extract a file (this isn\'t normal)... forcing an update to compensate')
+                    ForceUpdateDependencies = True
+            print(f'Extraction of {content_file_path} to {extract_path} completed.')
+
+        if ForceUpdateDependencies:
+            install_packages()
+            ForceUpdateDependencies = False
+    else:
+        print('CachedRVC.tar.gz not found. Proceeding to create an index of all current files...')
+        scan_and_write('/usr/', '/content/usr_files.csv')
+
+        install_packages()
+
+        scan_and_write('/usr/', '/content/usr_files_new.csv')
+        changed_files = compare_files('/content/usr_files.csv', '/content/usr_files_new.csv')
+
+        with tarfile.open('/content/CachedRVC.tar.gz', 'w:gz') as new_tar:
+            for file in changed_files:
+                new_tar.add(file)
+                print(f'Added to tar: {file}')
+
+        os.makedirs('/content/drive/MyDrive/RVC_Cached', exist_ok=True)
+        shutil.copy('/content/CachedRVC.tar.gz', '/content/drive/MyDrive/RVC_Cached/CachedRVC.tar.gz')
+        print('Updated CachedRVC.tar.gz copied to Google Drive.')
+        print('Dependencies fully up to date; future runs should be faster.')
+

utils/i18n.py

@@ -0,0 +1,28 @@
+import locale
+import json
+import os
+
+
+def load_language_list(language):
+    with open(f"./i18n/{language}.json", "r", encoding="utf-8") as f:
+        language_list = json.load(f)
+    return language_list
+
+
+class I18nAuto:
+    def __init__(self, language=None):
+        if language in ["Auto", None]:
+            language = "es_ES"
+        if not os.path.exists(f"./i18n/{language}.json"):
+            language = "es_ES"
+            language = "es_ES"
+        self.language = language
+        # print("Use Language:", language)
+        self.language_map = load_language_list(language)
+
+    def __call__(self, key):
+        return self.language_map.get(key, key)
+
+    def print(self):
+      # print("Use Language:", self.language)
+        print("")