all files (#1)

- all files (fdc77c35cee9a70d4bfa097f5e0e6d232db5df93)


Co-authored-by: Ciara <[email protected]>

README.md

@@ -1,13 +1,13 @@
 ---
-title: IP Adapter Instruct
-emoji: 🐢
+title: IP Test
+emoji: 😻
 colorFrom: purple
-colorTo: gray
+colorTo: pink
 sdk: gradio
 sdk_version: 4.40.0
 app_file: app.py
 pinned: false
-license: apache-2.0
 ---
+based on: https://huggingface.co/spaces/multimodalart/Ip-Adapter-FaceID <3
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,142 @@
+import torch
+import spaces
+from diffusers import StableDiffusionPipeline, DDIMScheduler, AutoencoderKL
+from transformers import AutoFeatureExtractor
+from ip_adapter.pipeline_stable_diffusion_extra_cfg import StableDiffusionPipelineCFG
+from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker
+
+from ip_adapter.ip_adapter_instruct import IPAdapterInstruct
+from huggingface_hub import hf_hub_download
+import gradio as gr
+import cv2
+
+base_model_path = "SG161222/Realistic_Vision_V4.0_noVAE"
+vae_model_path = "stabilityai/sd-vae-ft-mse"
+image_encoder_path = "laion/CLIP-ViT-H-14-laion2B-s32B-b79K"
+ip_ckpt = hf_hub_download(repo_id="CiaraRowles/IP-Adapter-Instruct", filename="ip-adapter-instruct-sd15.bin", repo_type="model")
+
+safety_model_id = "CompVis/stable-diffusion-safety-checker"
+safety_feature_extractor = AutoFeatureExtractor.from_pretrained(safety_model_id)
+safety_checker = StableDiffusionSafetyChecker.from_pretrained(safety_model_id)
+
+device = "cuda"
+
+noise_scheduler = DDIMScheduler(
+    num_train_timesteps=1000,
+    beta_start=0.00085,
+    beta_end=0.012,
+    beta_schedule="scaled_linear",
+    clip_sample=False,
+    set_alpha_to_one=False,
+    steps_offset=1,
+)
+vae = AutoencoderKL.from_pretrained(vae_model_path).to(dtype=torch.float16)
+pipe = StableDiffusionPipelineCFG.from_pretrained(
+    base_model_path,
+    scheduler=noise_scheduler,
+    vae=vae,
+    torch_dtype=torch.float16,
+    feature_extractor=safety_feature_extractor,
+    safety_checker=safety_checker
+).to(device)
+
+#pipe.load_lora_weights("h94/IP-Adapter-FaceID", weight_name="ip-adapter-faceid-plusv2_sd15_lora.safetensors")
+#pipe.fuse_lora()
+
+ip_model = IPAdapterInstruct(pipe, image_encoder_path, ip_ckpt, device,dtypein=torch.float16,num_tokens=16)
+
+cv2.setNumThreads(1)
+
+@spaces.GPU(enable_queue=True)
+def generate_image(images, prompt, negative_prompt,instruct_query, scale, nfaa_negative_prompt, progress=gr.Progress(track_tqdm=True)):
+    faceid_all_embeds = []
+    first_iteration = True
+    image = images
+    yield None
+    total_negative_prompt = f"{negative_prompt} {nfaa_negative_prompt}"
+    print("Generating normal")
+
+    # Calculate aspect ratio
+    aspect_ratio = image.width / image.height
+
+    # Set base_size (you can adjust this value as needed)
+    base_size = 512
+
+    # Calculate new width and height
+    if aspect_ratio > 1:  # Landscape
+        new_width = base_size
+        new_height = int(base_size / aspect_ratio)
+    else:  # Portrait or square
+        new_height = base_size
+        new_width = int(base_size * aspect_ratio)
+
+    # Ensure dimensions are multiples of 8 (required by some models)
+    new_width = (new_width // 8) * 8
+    new_height = (new_height // 8) * 8
+
+    image = ip_model.generate(
+        prompt=prompt,
+        negative_prompt=total_negative_prompt,
+        pil_image=image,
+        scale=scale,
+        width=new_width,
+        height=new_height,
+        num_inference_steps=30,
+        query=instruct_query
+    )
+    
+    yield image
+
+
+
+def swap_to_gallery(images):
+    return gr.update(value=images, visible=True), gr.update(visible=True), gr.update(visible=False)
+
+def remove_back_to_files():
+    return gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)
+css = '''
+h1{margin-bottom: 0 !important}
+'''
+with gr.Blocks(css=css) as demo:
+    gr.Markdown("# IP-Adapter-Instruct demo")
+    gr.Markdown("Demo for the [CiaraRowles/IP-Adapter-Instruct model](https://huggingface.co/CiaraRowles/IP-Adapter-Instruct)")
+    with gr.Row():
+        with gr.Column():
+            files = gr.Image(
+                label="Input image",
+                type="pil"
+            )
+            uploaded_files = gr.Gallery(label="Your image", visible=False, columns=5, rows=1, height=125)
+            with gr.Column(visible=False) as clear_button:
+                remove_and_reupload = gr.ClearButton(value="Remove and upload new ones", components=files, size="sm")
+            prompt = gr.Textbox(label="Prompt",
+                       info="Try something like 'a photo of a man/woman/person'",
+                       placeholder="A photo of a [man/woman/person]...")
+            instruct_query = gr.Dropdown(
+                label="Instruct Query",
+                choices=[
+                    "use everything from the image",
+                    "use the style",
+                    "use the colour",
+                    "use the pose",
+                    "use the composition",
+                    "use the face"
+                ],
+                value="use everything from the image"
+            )
+            negative_prompt = gr.Textbox(label="Negative Prompt", placeholder="low quality")
+            submit = gr.Button("Submit")
+            with gr.Accordion(open=False, label="Advanced Options"):
+                nfaa_negative_prompts = gr.Textbox(label="Appended Negative Prompts", info="Negative prompts to steer generations towards safe for all audiences outputs", value="naked, bikini, skimpy, scanty, bare skin, lingerie, swimsuit, exposed, see-through")    
+                scale = gr.Slider(label="Scale", value=0.8, step=0.1, minimum=0, maximum=2)
+        with gr.Column():
+            gallery = gr.Gallery(label="Generated Images")
+        
+        submit.click(fn=generate_image,
+                    inputs=[files, prompt, negative_prompt,instruct_query, scale, nfaa_negative_prompts],
+                    outputs=gallery)
+    
+    gr.Markdown("This demo includes extra features to mitigate the implicit bias of the model and prevent explicit usage of it to generate content with faces of people, including third parties, that is not safe for all audiences, including naked or semi-naked people.")
+    gr.Markdown("based on: https://huggingface.co/spaces/multimodalart/Ip-Adapter-FaceID")
+
+demo.launch()

requirements.txt

@@ -0,0 +1,9 @@
+diffusers==0.29.2
+transformers
+accelerate
+safetensors
+einops
+onnxruntime-gpu
+spaces==0.19.4
+opencv-python
+git+https://github.com/unity-research/IP-Adapter-Instruct.git