inital update

README.md

@@ -1,8 +1,8 @@
 ---
 title: JoyHallo
-emoji: 🏆
-colorFrom: indigo
-colorTo: blue
+emoji: 🏃
+colorFrom: gray
+colorTo: purple
 sdk: gradio
 sdk_version: 4.44.0
 app_file: app.py
@@ -10,4 +10,3 @@ pinned: false
 license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,88 @@
+import os
+import shutil
+from huggingface_hub import snapshot_download
+import gradio as gr
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+hallo_dir = snapshot_download(repo_id="fudan-generative-ai/hallo", local_dir="pretrained_models")
+joyhallo_dir = snapshot_download(repo_id="jdh-algo/JoyHallo-v1", local_dir="pretrained_models/joyhallo")
+wav_dir = snapshot_download(repo_id="TencentGameMate/chinese-wav2vec2-base", local_dir="pretrained_models/chinese-wav2vec2-base")
+print(hallo_dir, joyhallo_dir)
+print(os.listdir(hallo_dir))
+
+from scripts.inference import predict
+
+def run_inference(source_image, driving_audio, progress=gr.Progress(track_tqdm=True)):
+        
+    return predict(source_image, driving_audio, 1.0, 1.0, 1.0, 1.2)
+
+
+css = '''
+div#warning-ready {
+    background-color: #ecfdf5;
+    padding: 0 16px 16px;
+    margin: 20px 0;
+    color: #030303!important;
+}
+div#warning-ready > .gr-prose > h2, div#warning-ready > .gr-prose > p {
+    color: #057857!important;
+}
+div#warning-duplicate {
+    background-color: #ebf5ff;
+    padding: 0 16px 16px;
+    margin: 20px 0;
+    color: #030303!important;
+}
+div#warning-duplicate > .gr-prose > h2, div#warning-duplicate > .gr-prose > p {
+    color: #0f4592!important;
+}
+div#warning-duplicate strong {
+    color: #0f4592;
+}
+p.actions {
+    display: flex;
+    align-items: center;
+    margin: 20px 0;
+}
+div#warning-duplicate .actions a {
+    display: inline-block;
+    margin-right: 10px;
+}
+.dark #warning-duplicate {
+    background-color: #0c0c0c !important;
+    border: 1px solid white !important;
+}
+'''
+
+with gr.Blocks(css=css) as demo:
+    gr.Markdown("# JoyHallo: Digital human model for Mandarin")
+    gr.Markdown("Generate talking head avatars driven with Mandarin speech.")
+    gr.Markdown("""
+
+Data requirements:
+
+Image:
+1. Cropped to square shape.
+2. Face should be facing forward and occupy 50%-70% of the image area.
+
+Audio:
+1. Audio in wav format.
+2. Mandarin or English or mixed, with clear audio and suitable background music.
+                
+! Important: Too long audio will casue a very long processing time, please keep the audio length within 5s.
+                """)
+    with gr.Row():
+        with gr.Column():
+            avatar_face = gr.Image(type="filepath", label="Face")
+            driving_audio = gr.Audio(type="filepath", label="Driving audio")
+            generate = gr.Button("Generate")
+        with gr.Column():
+            output_video = gr.Video(label="Your talking head")
+
+    generate.click(
+        fn=run_inference,
+        inputs=[avatar_face, driving_audio],
+        outputs=output_video
+    )
+    
+demo.launch()

requirements.txt

@@ -0,0 +1,30 @@
+accelerate==0.28.0
+audio-separator==0.17.2
+av==12.1.0
+bitsandbytes==0.43.1
+decord==0.6.0
+diffusers==0.27.2
+einops==0.8.0
+insightface==0.7.3
+librosa==0.10.2.post1
+mediapipe[vision]==0.10.14
+mlflow==2.13.1
+moviepy==1.0.3
+numpy==1.26.4
+omegaconf==2.3.0
+onnx2torch==1.5.14
+onnx==1.16.1
+onnxruntime==1.18.0
+opencv-contrib-python==4.9.0.80
+opencv-python-headless==4.9.0.80
+opencv-python==4.9.0.80
+pillow==10.3.0
+setuptools==70.0.0
+torch==2.2.2
+torchvision==0.17.2
+tqdm==4.66.4
+transformers==4.39.2
+xformers==0.0.25.post1
+isort==5.13.2
+pylint==3.2.2
+pre-commit==3.7.1