Upload 4 files

README.md

@@ -1,15 +1,14 @@
----
-title: MagicAnimate
-emoji: 💃
-colorFrom: purple
-colorTo: purple
-sdk: gradio
-sdk_version: 4.10.0
-python_version: 3.11
-app_file: app.py
-models:
- - zcxu-eric/MagicAnimate
- - runwayml/stable-diffusion-v1-5
- - stabilityai/sd-vae-ft-mse
-pinned: false
+---
+title: MagicAnimate
+emoji: 💃
+colorFrom: purple
+colorTo: purple
+sdk: gradio
+sdk_version: 4.43.0
+app_file: app.py
+models:
+ - zcxu-eric/MagicAnimate
+ - runwayml/stable-diffusion-v1-5
+ - stabilityai/sd-vae-ft-mse
+pinned: false
 ---

app.py

@@ -1,114 +1,114 @@
-# Copyright 2023 ByteDance and/or its affiliates.
-#
-# Copyright (2023) MagicAnimate Authors
-#
-# ByteDance, its affiliates and licensors retain all intellectual
-# property and proprietary rights in and to this material, related
-# documentation and any modifications thereto. Any use, reproduction,
-# disclosure or distribution of this material and related documentation
-# without an express license agreement from ByteDance or
-# its affiliates is strictly prohibited.
-import argparse
-import imageio
-import numpy as np
-import gradio as gr
-import os
-from PIL import Image
-from subprocess import PIPE, run
-
-from demo.animate import MagicAnimate
-
-from huggingface_hub import snapshot_download
-
-snapshot_download(repo_id="stable-diffusion-v1-5/stable-diffusion-v1-5", local_dir="./stable-diffusion-v1-5", ignore_patterns=["*.safetensors"])
-snapshot_download(repo_id="stabilityai/sd-vae-ft-mse", local_dir="./sd-vae-ft-mse")
-snapshot_download(repo_id="zcxu-eric/MagicAnimate", local_dir="./MagicAnimate")
-
-is_spaces = True if "SPACE_ID" in os.environ else False
-true_for_shared_ui = False #This will be true only if you are in a shared UI
-if(is_spaces):
-    true_for_shared_ui = True if "zcxu-eric/magicanimate" in os.environ['SPACE_ID'] else False
-
-
-animator = MagicAnimate()
-
-def animate(reference_image, motion_sequence_state, seed=1, steps=25, guidance_scale=7.5):
-    return animator(reference_image, motion_sequence_state, seed, steps, guidance_scale)
-
-with gr.Blocks() as demo:
-
-    gr.HTML(
-        """
-        <div style="display: flex; justify-content: center; align-items: center; text-align: center;">
-        <a href="https://github.com/magic-research/magic-animate" style="margin-right: 20px; text-decoration: none; display: flex; align-items: center;">
-        </a>
-        <div>
-            <h1 >MagicAnimate: Temporally Consistent Human Image Animation using Diffusion Model</h1>
-            <h5 style="margin: 0;">If you like our project, please give us a star ✨ on Github for the latest update.</h5>
-            <div style="display: flex; justify-content: center; align-items: center; text-align: center;>
-                <a href="https://arxiv.org/abs/2311.16498"><img src="https://img.shields.io/badge/Arxiv-2311.16498-red"></a>
-                <a href='https://showlab.github.io/magicanimate'><img src='https://img.shields.io/badge/Project_Page-MagicAnimate-green' alt='Project Page'></a>
-                <a href='https://github.com/magic-research/magic-animate'><img src='https://img.shields.io/badge/Github-Code-blue'></a>
-            </div>
-        </div>
-        </div>
-        """)
-    animation = gr.Video(format="mp4", label="Animation Results", autoplay=True)
-
-    with gr.Row():
-        reference_image  = gr.Image(label="Reference Image")
-        motion_sequence  = gr.Video(format="mp4", label="Motion Sequence",max_length=5)
-
-        with gr.Column():
-            random_seed         = gr.Textbox(label="Random seed", value=1, info="default: -1")
-            sampling_steps      = gr.Textbox(label="Sampling steps", value=25, info="default: 25")
-            guidance_scale      = gr.Textbox(label="Guidance scale", value=7.5, info="default: 7.5")
-            submit              = gr.Button("Animate")
-
-    def read_video(video):
-        reader = imageio.get_reader(video)
-        fps = reader.get_meta_data()['fps']
-        return video
-
-    def read_image(image, size=512):
-        return np.array(Image.fromarray(image).resize((size, size)))
-
-    # when user uploads a new video
-    motion_sequence.upload(
-        read_video,
-        motion_sequence,
-        motion_sequence,
-        queue=False
-    )
-    # when `first_frame` is updated
-    reference_image.upload(
-        read_image,
-        reference_image,
-        reference_image,
-        queue=False
-    )
-    # when the `submit` button is clicked
-    submit.click(
-        animate,
-        [reference_image, motion_sequence, random_seed, sampling_steps, guidance_scale],
-        animation
-    )
-
-    # Examples
-    gr.Markdown("## Examples")
-    gr.Examples(
-        fn=animate,
-        examples=[
-            ["inputs/applications/source_image/monalisa.png", "inputs/applications/driving/densepose/running.mp4"],
-            ["inputs/applications/source_image/demo4.png", "inputs/applications/driving/densepose/demo4.mp4"],
-            ["inputs/applications/source_image/dalle2.jpeg", "inputs/applications/driving/densepose/running2.mp4"],
-            ["inputs/applications/source_image/dalle8.jpeg", "inputs/applications/driving/densepose/dancing2.mp4"],
-            ["inputs/applications/source_image/multi1_source.png", "inputs/applications/driving/densepose/multi_dancing.mp4"],
-        ],
-        inputs=[reference_image, motion_sequence],
-        outputs=animation,
-        cache_examples=true_for_shared_ui
-    )
-
-# demo.queue(max_size=15, api_open=False)
-demo.launch(share=True, show_api=False)
+# Copyright 2023 ByteDance and/or its affiliates.
+#
+# Copyright (2023) MagicAnimate Authors
+#
+# ByteDance, its affiliates and licensors retain all intellectual
+# property and proprietary rights in and to this material, related
+# documentation and any modifications thereto. Any use, reproduction,
+# disclosure or distribution of this material and related documentation
+# without an express license agreement from ByteDance or
+# its affiliates is strictly prohibited.
+import argparse
+import imageio
+import numpy as np
+import gradio as gr
+import os
+from PIL import Image
+from subprocess import PIPE, run
+
+from demo.animate import MagicAnimate
+
+from huggingface_hub import snapshot_download
+
+snapshot_download(repo_id="stable-diffusion-v1-5/stable-diffusion-v1-5", local_dir="./stable-diffusion-v1-5", ignore_patterns=["*.safetensors"])
+snapshot_download(repo_id="stabilityai/sd-vae-ft-mse", local_dir="./sd-vae-ft-mse")
+snapshot_download(repo_id="zcxu-eric/MagicAnimate", local_dir="./MagicAnimate")
+
+is_spaces = True if "SPACE_ID" in os.environ else False
+true_for_shared_ui = False #This will be true only if you are in a shared UI
+if(is_spaces):
+    true_for_shared_ui = True if "zcxu-eric/magicanimate" in os.environ['SPACE_ID'] else False
+
+
+animator = MagicAnimate()
+
+def animate(reference_image, motion_sequence_state, seed=1, steps=25, guidance_scale=7.5):
+    return animator(reference_image, motion_sequence_state, seed, steps, guidance_scale)
+
+with gr.Blocks() as demo:
+
+    gr.HTML(
+        """
+        <div style="display: flex; justify-content: center; align-items: center; text-align: center;">
+        <a href="https://github.com/magic-research/magic-animate" style="margin-right: 20px; text-decoration: none; display: flex; align-items: center;">
+        </a>
+        <div>
+            <h1 >MagicAnimate: Temporally Consistent Human Image Animation using Diffusion Model</h1>
+            <h5 style="margin: 0;">If you like our project, please give us a star ✨ on Github for the latest update.</h5>
+            <div style="display: flex; justify-content: center; align-items: center; text-align: center;>
+                <a href="https://arxiv.org/abs/2311.16498"><img src="https://img.shields.io/badge/Arxiv-2311.16498-red"></a>
+                <a href='https://showlab.github.io/magicanimate'><img src='https://img.shields.io/badge/Project_Page-MagicAnimate-green' alt='Project Page'></a>
+                <a href='https://github.com/magic-research/magic-animate'><img src='https://img.shields.io/badge/Github-Code-blue'></a>
+            </div>
+        </div>
+        </div>
+        """)
+    animation = gr.Video(format="mp4", label="Animation Results", autoplay=True)
+
+    with gr.Row():
+        reference_image  = gr.Image(label="Reference Image")
+        motion_sequence  = gr.Video(format="mp4", label="Motion Sequence",max_length=5)
+
+        with gr.Column():
+            random_seed         = gr.Textbox(label="Random seed", value=1, info="default: -1")
+            sampling_steps      = gr.Textbox(label="Sampling steps", value=25, info="default: 25")
+            guidance_scale      = gr.Textbox(label="Guidance scale", value=7.5, info="default: 7.5")
+            submit              = gr.Button("Animate")
+
+    def read_video(video):
+        reader = imageio.get_reader(video)
+        fps = reader.get_meta_data()['fps']
+        return video
+
+    def read_image(image, size=512):
+        return np.array(Image.fromarray(image).resize((size, size)))
+
+    # when user uploads a new video
+    motion_sequence.upload(
+        read_video,
+        motion_sequence,
+        motion_sequence,
+        queue=False
+    )
+    # when `first_frame` is updated
+    reference_image.upload(
+        read_image,
+        reference_image,
+        reference_image,
+        queue=False
+    )
+    # when the `submit` button is clicked
+    submit.click(
+        animate,
+        [reference_image, motion_sequence, random_seed, sampling_steps, guidance_scale],
+        animation
+    )
+
+    # Examples
+    gr.Markdown("## Examples")
+    gr.Examples(
+        fn=animate,
+        examples=[
+            ["inputs/applications/source_image/monalisa.png", "inputs/applications/driving/densepose/running.mp4"],
+            ["inputs/applications/source_image/demo4.png", "inputs/applications/driving/densepose/demo4.mp4"],
+            ["inputs/applications/source_image/dalle2.jpeg", "inputs/applications/driving/densepose/running2.mp4"],
+            ["inputs/applications/source_image/dalle8.jpeg", "inputs/applications/driving/densepose/dancing2.mp4"],
+            ["inputs/applications/source_image/multi1_source.png", "inputs/applications/driving/densepose/multi_dancing.mp4"],
+        ],
+        inputs=[reference_image, motion_sequence],
+        outputs=animation,
+        cache_examples=true_for_shared_ui
+    )
+
+# demo.queue(max_size=15, api_open=False)
+demo.launch(share=True, show_api=False)

demo/animate.py

@@ -64,7 +64,7 @@ class MagicAnimate():
             vae = AutoencoderKL.from_pretrained(config.pretrained_model_path, subfolder="vae")
 
         ### Load controlnet
-        controlnet   = ControlNetModel.from_pretrained(config.pretrained_controlnet_path)
+        controlnet = ControlNetModel.from_pretrained(config.pretrained_controlnet_path)
 
         vae.to(torch.float16)
         unet.to(torch.float16)

requirements.txt

@@ -1,127 +1,124 @@
-absl-py==1.4.0
-accelerate==0.22.0
-aiofiles==23.2.1
-aiohttp==3.8.5
-aiosignal==1.3.1
-altair==5.0.1
-annotated-types==0.5.0
-antlr4-python3-runtime==4.9.3
-anyio==3.7.1
-async-timeout==4.0.3
-attrs==23.1.0
-cachetools==5.3.1
-certifi==2023.7.22
-charset-normalizer==3.2.0
-click==8.1.7
-cmake==3.27.2
-contourpy==1.1.0
-cycler==0.11.0
-datasets==2.14.4
-dill==0.3.7
-einops==0.6.1
-exceptiongroup==1.1.3
-fastapi==0.103.0
-ffmpy==0.3.1
-filelock==3.12.2
-fonttools==4.42.1
-frozenlist==1.4.0
-fsspec==2023.6.0
-google-auth==2.22.0
-google-auth-oauthlib==1.0.0
-gradio==3.41.2
-gradio-client==0.5.0
-grpcio==1.57.0
-h11==0.14.0
-httpcore==0.17.3
-httpx==0.24.1
-huggingface-hub==0.16.4
-idna==3.4
-importlib-metadata==6.8.0
-importlib-resources==6.0.1
-jinja2==3.1.2
-joblib==1.3.2
-jsonschema==4.19.0
-jsonschema-specifications==2023.7.1
-kiwisolver==1.4.5
-lightning-utilities==0.9.0
-lit==16.0.6
-markdown==3.4.4
-markupsafe==2.1.3
-matplotlib==3.7.2
-mpmath==1.3.0
-multidict==6.0.4
-multiprocess==0.70.15
-networkx==3.1
-numpy==1.24.4
-nvidia-cublas-cu11==11.10.3.66
-nvidia-cuda-cupti-cu11==11.7.101
-nvidia-cuda-nvrtc-cu11==11.7.99
-nvidia-cuda-runtime-cu11==11.7.99
-nvidia-cudnn-cu11==8.5.0.96
-nvidia-cufft-cu11==10.9.0.58
-nvidia-curand-cu11==10.2.10.91
-nvidia-cusolver-cu11==11.4.0.1
-nvidia-cusparse-cu11==11.7.4.91
-nvidia-nccl-cu11==2.14.3
-nvidia-nvtx-cu11==11.7.91
-oauthlib==3.2.2
-omegaconf==2.3.0
-opencv-python==4.8.0.76
-orjson==3.9.5
-pandas==2.0.3
-pillow==9.5.0
-pkgutil-resolve-name==1.3.10
-protobuf==4.24.2
-psutil==5.9.5
-pyarrow==13.0.0
-pyasn1==0.5.0
-pyasn1-modules==0.3.0
-pydantic==2.3.0
-pydantic-core==2.6.3
-pydub==0.25.1
-pyparsing==3.0.9
-python-multipart==0.0.6
-pytorch-lightning==2.0.7
-pytz==2023.3
-pyyaml==6.0.1
-referencing==0.30.2
-regex==2023.8.8
-requests==2.31.0
-requests-oauthlib==1.3.1
-rpds-py==0.9.2
-rsa==4.9
-safetensors==0.3.3
-semantic-version==2.10.0
-sniffio==1.3.0
-starlette==0.27.0
-sympy==1.12
-tensorboard==2.14.0
-tensorboard-data-server==0.7.1
-tokenizers==0.13.3
-toolz==0.12.0
-torchmetrics==1.1.0
-tqdm==4.66.1
-transformers==4.32.0
-triton==2.0.0
-tzdata==2023.3
-urllib3==1.26.16
-uvicorn==0.23.2
-websockets==11.0.3
-werkzeug==2.3.7
-xxhash==3.3.0
-yarl==1.9.2
-zipp==3.16.2
-decord
-imageio==2.9.0
-imageio-ffmpeg==0.4.3
-timm
-scipy
-scikit-image
-av
-imgaug
-lpips
-ffmpeg-python
-torch==2.0.1
-torchvision==0.15.2
-xformers==0.0.22
-diffusers==0.21.4
+absl-py==1.4.0
+accelerate==0.22.0
+aiofiles==23.2.1
+aiohttp==3.8.5
+aiosignal==1.3.1
+altair==5.0.1
+annotated-types==0.5.0
+antlr4-python3-runtime==4.9.3
+anyio==3.7.1
+async-timeout==4.0.3
+attrs==23.1.0
+cachetools==5.3.1
+certifi==2023.7.22
+charset-normalizer==3.2.0
+click==8.1.7
+cmake==3.27.2
+contourpy==1.1.0
+cycler==0.11.0
+datasets==2.14.4
+dill==0.3.7
+einops==0.6.1
+exceptiongroup==1.1.3
+fastapi==0.103.0
+ffmpy==0.3.1
+filelock==3.12.2
+fonttools==4.42.1
+frozenlist==1.4.0
+fsspec==2023.6.0
+google-auth==2.22.0
+google-auth-oauthlib==1.0.0
+grpcio==1.57.0
+h11==0.14.0
+httpcore
+httpx
+huggingface-hub
+idna==3.4
+importlib-metadata==6.8.0
+importlib-resources==6.0.1
+jinja2==3.1.2
+joblib==1.3.2
+jsonschema==4.19.0
+jsonschema-specifications==2023.7.1
+kiwisolver==1.4.5
+lightning-utilities==0.9.0
+lit==16.0.6
+markdown==3.4.4
+markupsafe==2.1.3
+matplotlib==3.7.2
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.15
+networkx==3.1
+numpy==1.24.4
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+nvidia-cufft-cu11==10.9.0.58
+nvidia-curand-cu11==10.2.10.91
+nvidia-cusolver-cu11==11.4.0.1
+nvidia-cusparse-cu11==11.7.4.91
+nvidia-nccl-cu11==2.14.3
+nvidia-nvtx-cu11==11.7.91
+oauthlib==3.2.2
+omegaconf==2.3.0
+opencv-python==4.8.0.76
+orjson==3.9.5
+pandas==2.0.3
+pillow==9.5.0
+pkgutil-resolve-name==1.3.10
+protobuf==4.24.2
+psutil==5.9.5
+pyarrow==13.0.0
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pydantic==2.3.0
+pydantic-core==2.6.3
+pydub==0.25.1
+pyparsing==3.0.9
+python-multipart==0.0.6
+pytorch-lightning==2.0.7
+pytz==2023.3
+pyyaml==6.0.1
+referencing==0.30.2
+regex
+requests
+requests-oauthlib
+rpds-py==0.9.2
+rsa==4.9
+safetensors==0.3.3
+semantic-version==2.10.0
+sniffio==1.3.0
+starlette==0.27.0
+sympy==1.12
+tensorboard==2.14.0
+tensorboard-data-server==0.7.1
+tokenizers==0.13.3
+toolz==0.12.0
+torchmetrics==1.1.0
+tqdm
+transformers==4.32.0
+triton==2.0.0
+tzdata==2023.3
+urllib3==1.26.16
+uvicorn==0.23.2
+websockets==11.0.3
+werkzeug==2.3.7
+xxhash==3.3.0
+yarl==1.9.2
+zipp==3.16.2
+decord
+imageio==2.9.0
+imageio-ffmpeg==0.4.3
+timm
+scipy
+scikit-image
+av
+imgaug
+lpips
+ffmpeg-python
+torch==2.0.1
+torchvision==0.15.2
+diffusers==0.21.4