HighCWu commited on
Commit
7a1ec93
1 Parent(s): 5fe3b59

make the app runable

Browse files
.gitignore CHANGED
@@ -160,3 +160,5 @@ cython_debug/
160
  # and can be added to the global gitignore or merged into this file. For a more nuclear
161
  # option (not recommended) you can uncomment the following to ignore the entire idea folder.
162
  #.idea/
 
 
 
160
  # and can be added to the global gitignore or merged into this file. For a more nuclear
161
  # option (not recommended) you can uncomment the following to ignore the entire idea folder.
162
  #.idea/
163
+
164
+ my_*
.vscode/settings.json CHANGED
@@ -3,7 +3,7 @@
3
  "editor.defaultFormatter": "ms-python.black-formatter",
4
  "editor.formatOnType": true,
5
  "editor.codeActionsOnSave": {
6
- "source.organizeImports": true
7
  }
8
  },
9
  "black-formatter.args": [
 
3
  "editor.defaultFormatter": "ms-python.black-formatter",
4
  "editor.formatOnType": true,
5
  "editor.codeActionsOnSave": {
6
+ "source.organizeImports": "explicit"
7
  }
8
  },
9
  "black-formatter.args": [
LICENSE CHANGED
@@ -1,6 +1,6 @@
1
  MIT License
2
 
3
- Copyright (c) 2023 hysts
4
 
5
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
  of this software and associated documentation files (the "Software"), to deal
 
1
  MIT License
2
 
3
+ Copyright (c) 2024 wuhecong
4
 
5
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
  of this software and associated documentation files (the "Software"), to deal
LICENSE.hysts ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2023 hysts
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
app.py CHANGED
@@ -1,5 +1,4 @@
1
  #!/usr/bin/env python
2
-
3
  from __future__ import annotations
4
 
5
  import spaces
@@ -15,6 +14,7 @@ from app_ip2p import create_demo as create_demo_ip2p
15
  from app_lineart import create_demo as create_demo_lineart
16
  from app_mlsd import create_demo as create_demo_mlsd
17
  from app_normal import create_demo as create_demo_normal
 
18
  from app_openpose import create_demo as create_demo_openpose
19
  from app_scribble import create_demo as create_demo_scribble
20
  from app_scribble_interactive import create_demo as create_demo_scribble_interactive
@@ -26,12 +26,17 @@ from model import Model
26
  from settings import ALLOW_CHANGING_BASE_MODEL, DEFAULT_MODEL_ID, SHOW_DUPLICATE_BUTTON
27
 
28
  DESCRIPTION = r"""
29
- # ControlLoRA Version 3: LoRA Is All You Need to Control the Spatial Information of Stable Diffusion
30
-
31
  <center>
 
 
32
  <a href="https://huggingface.co/HighCWu/control-lora-v3">[Models]</a>
33
  <a href="https://github.com/HighCWu/control-lora-v3">[Github]</a>
34
  </center>
 
 
 
 
 
35
  """
36
 
37
  model = Model(base_model_id=DEFAULT_MODEL_ID, task_name="Canny")
@@ -61,8 +66,10 @@ with gr.Blocks(css="style.css") as demo:
61
  create_demo_segmentation(spaces.GPU(model.process_segmentation))
62
  with gr.TabItem("Depth"):
63
  create_demo_depth(spaces.GPU(model.process_depth))
64
- with gr.TabItem("Normal map"):
65
- create_demo_normal(spaces.GPU(model.process_normal))
 
 
66
  # with gr.TabItem("Lineart"):
67
  # create_demo_lineart(spaces.GPU(model.process_lineart))
68
  # with gr.TabItem("Content Shuffle"):
 
1
  #!/usr/bin/env python
 
2
  from __future__ import annotations
3
 
4
  import spaces
 
14
  from app_lineart import create_demo as create_demo_lineart
15
  from app_mlsd import create_demo as create_demo_mlsd
16
  from app_normal import create_demo as create_demo_normal
17
+ from app_normal_old import create_demo as create_demo_normal_old
18
  from app_openpose import create_demo as create_demo_openpose
19
  from app_scribble import create_demo as create_demo_scribble
20
  from app_scribble_interactive import create_demo as create_demo_scribble_interactive
 
26
  from settings import ALLOW_CHANGING_BASE_MODEL, DEFAULT_MODEL_ID, SHOW_DUPLICATE_BUTTON
27
 
28
  DESCRIPTION = r"""
 
 
29
  <center>
30
+ <h1>ControlLoRA Version 3: LoRA Is All You Need to Control the Spatial Information of Stable Diffusion</h1>
31
+
32
  <a href="https://huggingface.co/HighCWu/control-lora-v3">[Models]</a>
33
  <a href="https://github.com/HighCWu/control-lora-v3">[Github]</a>
34
  </center>
35
+
36
+ ***Note:*** I used a high learning rate and a short number of steps for training, and the dataset was also generated,
37
+ so the generation results may not be very good.
38
+ It is recommended that researchers use real data, lower learning and longer training steps to train to achieve better generation results.
39
+
40
  """
41
 
42
  model = Model(base_model_id=DEFAULT_MODEL_ID, task_name="Canny")
 
66
  create_demo_segmentation(spaces.GPU(model.process_segmentation))
67
  with gr.TabItem("Depth"):
68
  create_demo_depth(spaces.GPU(model.process_depth))
69
+ # with gr.TabItem("Normal map"):
70
+ # create_demo_normal(spaces.GPU(model.process_normal))
71
+ with gr.TabItem("Normal map (old)"):
72
+ create_demo_normal_old(spaces.GPU(model.process_normal_old))
73
  # with gr.TabItem("Lineart"):
74
  # create_demo_lineart(spaces.GPU(model.process_lineart))
75
  # with gr.TabItem("Content Shuffle"):
app_canny.py CHANGED
@@ -37,7 +37,7 @@ def create_demo(process):
37
  label="Canny high threshold", minimum=1, maximum=255, value=200, step=1
38
  )
39
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
40
- guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
41
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
42
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
43
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
 
37
  label="Canny high threshold", minimum=1, maximum=255, value=200, step=1
38
  )
39
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
40
+ guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
41
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
42
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
43
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
app_depth.py CHANGED
@@ -37,7 +37,7 @@ def create_demo(process):
37
  label="Preprocess resolution", minimum=128, maximum=512, value=384, step=1
38
  )
39
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
40
- guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
41
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
42
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
43
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
 
37
  label="Preprocess resolution", minimum=128, maximum=512, value=384, step=1
38
  )
39
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
40
+ guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
41
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
42
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
43
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
app_ip2p.py CHANGED
@@ -31,7 +31,7 @@ def create_demo(process):
31
  step=256,
32
  )
33
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
34
- guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
35
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
36
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
37
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
 
31
  step=256,
32
  )
33
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
34
+ guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
35
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
36
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
37
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
app_lineart.py CHANGED
@@ -47,7 +47,7 @@ def create_demo(process):
47
  label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
48
  )
49
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
50
- guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
51
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
52
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
53
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
 
47
  label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
48
  )
49
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
50
+ guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
51
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
52
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
53
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
app_mlsd.py CHANGED
@@ -40,7 +40,7 @@ def create_demo(process):
40
  label="Hough distance threshold (MLSD)", minimum=0.01, maximum=20.0, value=0.1, step=0.01
41
  )
42
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
43
- guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
44
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
45
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
46
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
 
40
  label="Hough distance threshold (MLSD)", minimum=0.01, maximum=20.0, value=0.1, step=0.01
41
  )
42
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
43
+ guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
44
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
45
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
46
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
app_normal.py CHANGED
@@ -37,7 +37,7 @@ def create_demo(process):
37
  label="Preprocess resolution", minimum=128, maximum=512, value=384, step=1
38
  )
39
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
40
- guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
41
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
42
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
43
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
 
37
  label="Preprocess resolution", minimum=128, maximum=512, value=384, step=1
38
  )
39
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
40
+ guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
41
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
42
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
43
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
app_normal_old.py ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ import gradio as gr
4
+
5
+ from settings import (
6
+ DEFAULT_IMAGE_RESOLUTION,
7
+ DEFAULT_NUM_IMAGES,
8
+ MAX_IMAGE_RESOLUTION,
9
+ MAX_NUM_IMAGES,
10
+ MAX_SEED,
11
+ )
12
+ from utils import randomize_seed_fn
13
+
14
+
15
+ def create_demo(process):
16
+ with gr.Blocks() as demo:
17
+ with gr.Row():
18
+ with gr.Column():
19
+ image = gr.Image()
20
+ prompt = gr.Textbox(label="Prompt")
21
+ run_button = gr.Button("Run")
22
+ with gr.Accordion("Advanced options", open=False):
23
+ preprocessor_name = gr.Radio(
24
+ label="Preprocessor", choices=["Normal", "None"], type="value", value="Normal"
25
+ )
26
+ num_samples = gr.Slider(
27
+ label="Images", minimum=1, maximum=MAX_NUM_IMAGES, value=DEFAULT_NUM_IMAGES, step=1
28
+ )
29
+ image_resolution = gr.Slider(
30
+ label="Image resolution",
31
+ minimum=256,
32
+ maximum=MAX_IMAGE_RESOLUTION,
33
+ value=DEFAULT_IMAGE_RESOLUTION,
34
+ step=256,
35
+ )
36
+ preprocess_resolution = gr.Slider(
37
+ label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
38
+ )
39
+ num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
40
+ guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
41
+ seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
42
+ randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
43
+ a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
44
+ n_prompt = gr.Textbox(
45
+ label="Negative prompt",
46
+ value="longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality",
47
+ )
48
+ with gr.Column():
49
+ result = gr.Gallery(label="Output", show_label=False, columns=2, object_fit="scale-down")
50
+ inputs = [
51
+ image,
52
+ prompt,
53
+ a_prompt,
54
+ n_prompt,
55
+ num_samples,
56
+ image_resolution,
57
+ preprocess_resolution,
58
+ num_steps,
59
+ guidance_scale,
60
+ seed,
61
+ preprocessor_name,
62
+ ]
63
+ prompt.submit(
64
+ fn=randomize_seed_fn,
65
+ inputs=[seed, randomize_seed],
66
+ outputs=seed,
67
+ queue=False,
68
+ api_name=False,
69
+ ).then(
70
+ fn=process,
71
+ inputs=inputs,
72
+ outputs=result,
73
+ api_name=False,
74
+ )
75
+ run_button.click(
76
+ fn=randomize_seed_fn,
77
+ inputs=[seed, randomize_seed],
78
+ outputs=seed,
79
+ queue=False,
80
+ api_name=False,
81
+ ).then(
82
+ fn=process,
83
+ inputs=inputs,
84
+ outputs=result,
85
+ api_name="normal",
86
+ )
87
+ return demo
88
+
89
+
90
+ if __name__ == "__main__":
91
+ from model import Model
92
+
93
+ model = Model(task_name="NormalBae")
94
+ demo = create_demo(model.process_normal)
95
+ demo.queue().launch()
app_openpose.py CHANGED
@@ -37,7 +37,7 @@ def create_demo(process):
37
  label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
38
  )
39
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
40
- guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
41
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
42
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
43
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
 
37
  label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
38
  )
39
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
40
+ guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
41
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
42
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
43
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
app_scribble.py CHANGED
@@ -37,7 +37,7 @@ def create_demo(process):
37
  label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
38
  )
39
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
40
- guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
41
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
42
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
43
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
 
37
  label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
38
  )
39
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
40
+ guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
41
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
42
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
43
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
app_scribble_interactive.py CHANGED
@@ -51,7 +51,7 @@ def create_demo(process):
51
  step=256,
52
  )
53
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
54
- guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
55
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
56
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
57
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
 
51
  step=256,
52
  )
53
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
54
+ guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
55
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
56
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
57
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
app_segmentation.py CHANGED
@@ -21,7 +21,7 @@ def create_demo(process):
21
  run_button = gr.Button("Run")
22
  with gr.Accordion("Advanced options", open=False):
23
  preprocessor_name = gr.Radio(
24
- label="Preprocessor", choices=["UPerNet", "None"], type="value", value="UPerNet"
25
  )
26
  num_samples = gr.Slider(
27
  label="Number of images", minimum=1, maximum=MAX_NUM_IMAGES, value=DEFAULT_NUM_IMAGES, step=1
@@ -37,7 +37,7 @@ def create_demo(process):
37
  label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
38
  )
39
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
40
- guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
41
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
42
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
43
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
 
21
  run_button = gr.Button("Run")
22
  with gr.Accordion("Advanced options", open=False):
23
  preprocessor_name = gr.Radio(
24
+ label="Preprocessor", choices=["OneFormer", "UPerNet", "None"], type="value", value="OneFormer"
25
  )
26
  num_samples = gr.Slider(
27
  label="Number of images", minimum=1, maximum=MAX_NUM_IMAGES, value=DEFAULT_NUM_IMAGES, step=1
 
37
  label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
38
  )
39
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
40
+ guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
41
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
42
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
43
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
app_shuffle.py CHANGED
@@ -34,7 +34,7 @@ def create_demo(process):
34
  step=256,
35
  )
36
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
37
- guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
38
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
39
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
40
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
 
34
  step=256,
35
  )
36
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
37
+ guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
38
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
39
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
40
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
app_softedge.py CHANGED
@@ -46,7 +46,7 @@ def create_demo(process):
46
  label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
47
  )
48
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
49
- guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
50
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
51
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
52
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
 
46
  label="Preprocess resolution", minimum=128, maximum=512, value=512, step=1
47
  )
48
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
49
+ guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
50
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
51
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
52
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
app_tile.py CHANGED
@@ -31,7 +31,7 @@ def create_demo(process):
31
  step=256,
32
  )
33
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
34
- guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=9.0, step=0.1)
35
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
36
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
37
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
 
31
  step=256,
32
  )
33
  num_steps = gr.Slider(label="Number of steps", minimum=1, maximum=100, value=20, step=1)
34
+ guidance_scale = gr.Slider(label="Guidance scale", minimum=0.1, maximum=30.0, value=7.5, step=0.1)
35
  seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
36
  randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
37
  a_prompt = gr.Textbox(label="Additional prompt", value="best quality, extremely detailed")
depth_estimator.py CHANGED
@@ -1,3 +1,4 @@
 
1
  import numpy as np
2
  import PIL.Image
3
  from controlnet_aux.util import HWC3
@@ -8,7 +9,8 @@ from cv_utils import resize_image
8
 
9
  class DepthEstimator:
10
  def __init__(self):
11
- self.model = pipeline("depth-estimation")
 
12
 
13
  def __call__(self, image: np.ndarray, **kwargs) -> PIL.Image.Image:
14
  detect_resolution = kwargs.pop("detect_resolution", 512)
 
1
+ import torch
2
  import numpy as np
3
  import PIL.Image
4
  from controlnet_aux.util import HWC3
 
9
 
10
  class DepthEstimator:
11
  def __init__(self):
12
+ self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
13
+ self.model = pipeline("depth-estimation", device=self.device)
14
 
15
  def __call__(self, image: np.ndarray, **kwargs) -> PIL.Image.Image:
16
  detect_resolution = kwargs.pop("detect_resolution", 512)
image_segmentor.py CHANGED
@@ -3,15 +3,17 @@ import numpy as np
3
  import PIL.Image
4
  import torch
5
  from controlnet_aux.util import HWC3, ade_palette
6
- from transformers import AutoImageProcessor, UperNetForSemanticSegmentation
7
 
8
  from cv_utils import resize_image
9
 
10
 
11
  class ImageSegmentor:
12
  def __init__(self):
 
13
  self.image_processor = AutoImageProcessor.from_pretrained("openmmlab/upernet-convnext-small")
14
  self.image_segmentor = UperNetForSemanticSegmentation.from_pretrained("openmmlab/upernet-convnext-small")
 
15
 
16
  @torch.inference_mode()
17
  def __call__(self, image: np.ndarray, **kwargs) -> PIL.Image.Image:
@@ -22,8 +24,36 @@ class ImageSegmentor:
22
  image = PIL.Image.fromarray(image)
23
 
24
  pixel_values = self.image_processor(image, return_tensors="pt").pixel_values
25
- outputs = self.image_segmentor(pixel_values)
26
- seg = self.image_processor.post_process_semantic_segmentation(outputs, target_sizes=[image.size[::-1]])[0]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8)
28
  for label, color in enumerate(ade_palette()):
29
  color_seg[seg == label, :] = color
 
3
  import PIL.Image
4
  import torch
5
  from controlnet_aux.util import HWC3, ade_palette
6
+ from transformers import AutoImageProcessor, UperNetForSemanticSegmentation, OneFormerProcessor, OneFormerForUniversalSegmentation
7
 
8
  from cv_utils import resize_image
9
 
10
 
11
  class ImageSegmentor:
12
  def __init__(self):
13
+ self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
14
  self.image_processor = AutoImageProcessor.from_pretrained("openmmlab/upernet-convnext-small")
15
  self.image_segmentor = UperNetForSemanticSegmentation.from_pretrained("openmmlab/upernet-convnext-small")
16
+ self.image_segmentor.to(self.device)
17
 
18
  @torch.inference_mode()
19
  def __call__(self, image: np.ndarray, **kwargs) -> PIL.Image.Image:
 
24
  image = PIL.Image.fromarray(image)
25
 
26
  pixel_values = self.image_processor(image, return_tensors="pt").pixel_values
27
+ outputs = self.image_segmentor(pixel_values.to(self.device))
28
+ seg = self.image_processor.post_process_semantic_segmentation(outputs, target_sizes=[image.size[::-1]])[0].cpu()
29
+ color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8)
30
+ for label, color in enumerate(ade_palette()):
31
+ color_seg[seg == label, :] = color
32
+ color_seg = color_seg.astype(np.uint8)
33
+
34
+ color_seg = resize_image(color_seg, resolution=image_resolution, interpolation=cv2.INTER_NEAREST)
35
+ return PIL.Image.fromarray(color_seg)
36
+
37
+
38
+ class ImageSegmentorOneFormer:
39
+ def __init__(self):
40
+ self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
41
+ self.image_processor = OneFormerProcessor.from_pretrained("shi-labs/oneformer_ade20k_swin_tiny")
42
+ self.image_segmentor = OneFormerForUniversalSegmentation.from_pretrained("shi-labs/oneformer_ade20k_swin_tiny")
43
+ self.image_segmentor.to(self.device)
44
+
45
+ @torch.inference_mode()
46
+ def __call__(self, image: np.ndarray, **kwargs) -> PIL.Image.Image:
47
+ detect_resolution = kwargs.pop("detect_resolution", 512)
48
+ image_resolution = kwargs.pop("image_resolution", 512)
49
+ image = HWC3(image)
50
+ image = resize_image(image, resolution=detect_resolution)
51
+ image = PIL.Image.fromarray(image)
52
+
53
+ inputs = self.image_processor(image, ["semantic"], return_tensors="pt")
54
+ inputs = {k: v.to(self.device) if isinstance(v, torch.Tensor) else v for k, v in inputs.items()}
55
+ outputs = self.image_segmentor(**inputs)
56
+ seg = self.image_processor.post_process_semantic_segmentation(outputs, target_sizes=[image.size[::-1]])[0].cpu()
57
  color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8)
58
  for label, color in enumerate(ade_palette()):
59
  color_seg[seg == label, :] = color
model.py CHANGED
@@ -22,9 +22,8 @@ CONTROL_LORA_V3_MODEL_IDS = OrderedDict([
22
  ("Openpose", "sd-control-lora-v3-pose-half-rank128-conv_in-rank128"),
23
  ("Canny", "sd-control-lora-v3-canny-half_skip_attn-rank16-conv_in-rank64"),
24
  ("segmentation", "sd-control-lora-v3-segmentation-half_skip_attn-rank128-conv_in-rank128"),
25
- ("depth", "lllyasviel/control_v11f1p_sd15_depth"),
26
- ("NormalBae", "sd-control-lora-v3-normal-half-rank32-conv_in-rank128"),
27
  ("depth", "sd-control-lora-v3-depth-half-rank8-conv_in-rank128"),
 
28
  ("Tile", "sd-control-lora-v3-tile-half_skip_attn-rank16-conv_in-rank64"),
29
  ])
30
 
@@ -37,6 +36,14 @@ class Model:
37
  self.pipe: StableDiffusionControlLoraV3Pipeline = self.load_pipe(base_model_id, task_name)
38
  self.preprocessor = Preprocessor()
39
 
 
 
 
 
 
 
 
 
40
  def load_pipe(self, base_model_id: str, task_name) -> StableDiffusionControlLoraV3Pipeline:
41
  if (
42
  base_model_id == self.base_model_id
@@ -55,6 +62,7 @@ class Model:
55
  )
56
  for _task_name, subfolder in CONTROL_LORA_V3_MODEL_IDS.items():
57
  pipe.load_lora_weights("HighCWu/control-lora-v3", adapter_name=_task_name, subfolder=subfolder)
 
58
  pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
59
  if self.device.type == "cuda":
60
  pipe.enable_xformers_memory_efficient_attention()
@@ -484,6 +492,52 @@ class Model:
484
  )
485
  return [control_image] + results
486
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
487
  @torch.inference_mode()
488
  def process_normal(
489
  self,
 
22
  ("Openpose", "sd-control-lora-v3-pose-half-rank128-conv_in-rank128"),
23
  ("Canny", "sd-control-lora-v3-canny-half_skip_attn-rank16-conv_in-rank64"),
24
  ("segmentation", "sd-control-lora-v3-segmentation-half_skip_attn-rank128-conv_in-rank128"),
 
 
25
  ("depth", "sd-control-lora-v3-depth-half-rank8-conv_in-rank128"),
26
+ ("Normal", "sd-control-lora-v3-normal-half-rank32-conv_in-rank128"),
27
  ("Tile", "sd-control-lora-v3-tile-half_skip_attn-rank16-conv_in-rank64"),
28
  ])
29
 
 
36
  self.pipe: StableDiffusionControlLoraV3Pipeline = self.load_pipe(base_model_id, task_name)
37
  self.preprocessor = Preprocessor()
38
 
39
+ # preload
40
+ preprocessor = self.preprocessor
41
+ preprocessor.load("Openpose")
42
+ preprocessor.load("Canny")
43
+ preprocessor.load("OneFormer"); preprocessor.load("UPerNet") # segmentation
44
+ preprocessor.load("DPT") # depth
45
+ preprocessor.load("Midas") # normal (old)
46
+
47
  def load_pipe(self, base_model_id: str, task_name) -> StableDiffusionControlLoraV3Pipeline:
48
  if (
49
  base_model_id == self.base_model_id
 
62
  )
63
  for _task_name, subfolder in CONTROL_LORA_V3_MODEL_IDS.items():
64
  pipe.load_lora_weights("HighCWu/control-lora-v3", adapter_name=_task_name, subfolder=subfolder)
65
+ pipe.unet.set_adapter(task_name)
66
  pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
67
  if self.device.type == "cuda":
68
  pipe.enable_xformers_memory_efficient_attention()
 
492
  )
493
  return [control_image] + results
494
 
495
+ @torch.inference_mode()
496
+ def process_normal_old(
497
+ self,
498
+ image: np.ndarray,
499
+ prompt: str,
500
+ additional_prompt: str,
501
+ negative_prompt: str,
502
+ num_images: int,
503
+ image_resolution: int,
504
+ preprocess_resolution: int,
505
+ num_steps: int,
506
+ guidance_scale: float,
507
+ seed: int,
508
+ preprocessor_name: str,
509
+ ) -> list[PIL.Image.Image]:
510
+ if image is None:
511
+ raise ValueError
512
+ if image_resolution > MAX_IMAGE_RESOLUTION:
513
+ raise ValueError
514
+ if num_images > MAX_NUM_IMAGES:
515
+ raise ValueError
516
+
517
+ if preprocessor_name == "None":
518
+ image = HWC3(image)
519
+ image = resize_image(image, resolution=image_resolution)
520
+ control_image = PIL.Image.fromarray(image)
521
+ else:
522
+ self.preprocessor.load("Midas")
523
+ control_image = self.preprocessor(
524
+ image=image,
525
+ image_resolution=image_resolution,
526
+ detect_resolution=preprocess_resolution,
527
+ depth_and_normal=True
528
+ )
529
+ self.load_controlnet_weight("Normal")
530
+ results = self.run_pipe(
531
+ prompt=self.get_prompt(prompt, additional_prompt),
532
+ negative_prompt=negative_prompt,
533
+ control_image=control_image,
534
+ num_images=num_images,
535
+ num_steps=num_steps,
536
+ guidance_scale=guidance_scale,
537
+ seed=seed,
538
+ )
539
+ return [control_image] + results
540
+
541
  @torch.inference_mode()
542
  def process_normal(
543
  self,
pipeline.py CHANGED
@@ -896,7 +896,10 @@ class StableDiffusionControlLoraV3Pipeline(
896
  kwargs["weight_name"] = kwargs.pop("weight_name", "pytorch_lora_weights.safetensors")
897
 
898
  if adapter_name is not None and adapter_name not in unet.extra_condition_names:
899
- return super().load_lora_weights(pretrained_model_name_or_path_or_dict, adapter_name, **kwargs)
 
 
 
900
 
901
  if not isinstance(pretrained_model_name_or_path_or_dict, list):
902
  pretrained_model_name_or_path_or_dict = [pretrained_model_name_or_path_or_dict] * num_condition_names
 
896
  kwargs["weight_name"] = kwargs.pop("weight_name", "pytorch_lora_weights.safetensors")
897
 
898
  if adapter_name is not None and adapter_name not in unet.extra_condition_names:
899
+ unet._hf_peft_config_loaded = True
900
+ super().load_lora_weights(pretrained_model_name_or_path_or_dict, adapter_name, **kwargs)
901
+ unet.set_adapter(adapter_name)
902
+ return
903
 
904
  if not isinstance(pretrained_model_name_or_path_or_dict, list):
905
  pretrained_model_name_or_path_or_dict = [pretrained_model_name_or_path_or_dict] * num_condition_names
preprocessor.py CHANGED
@@ -19,7 +19,7 @@ from controlnet_aux.util import HWC3
19
 
20
  from cv_utils import resize_image
21
  from depth_estimator import DepthEstimator
22
- from image_segmentor import ImageSegmentor
23
 
24
 
25
  class Preprocessor:
@@ -27,11 +27,16 @@ class Preprocessor:
27
 
28
  def __init__(self):
29
  self.model = None
 
30
  self.name = ""
31
 
32
  def load(self, name: str) -> None:
33
  if name == self.name:
34
  return
 
 
 
 
35
  if name == "HED":
36
  self.model = HEDdetector.from_pretrained(self.MODEL_ID)
37
  elif name == "Midas":
@@ -56,12 +61,15 @@ class Preprocessor:
56
  self.model = DepthEstimator()
57
  elif name == "UPerNet":
58
  self.model = ImageSegmentor()
 
 
59
  else:
60
  raise ValueError
61
- if torch.cuda.is_available():
62
- torch.cuda.empty_cache()
63
- gc.collect()
64
  self.name = name
 
65
 
66
  def __call__(self, image: PIL.Image.Image, **kwargs) -> PIL.Image.Image:
67
  if self.name == "Canny":
@@ -79,6 +87,8 @@ class Preprocessor:
79
  image = HWC3(image)
80
  image = resize_image(image, resolution=detect_resolution)
81
  image = self.model(image, **kwargs)
 
 
82
  image = HWC3(image)
83
  image = resize_image(image, resolution=image_resolution)
84
  return PIL.Image.fromarray(image)
 
19
 
20
  from cv_utils import resize_image
21
  from depth_estimator import DepthEstimator
22
+ from image_segmentor import ImageSegmentor, ImageSegmentorOneFormer
23
 
24
 
25
  class Preprocessor:
 
27
 
28
  def __init__(self):
29
  self.model = None
30
+ self.models = {}
31
  self.name = ""
32
 
33
  def load(self, name: str) -> None:
34
  if name == self.name:
35
  return
36
+ if name in self.models:
37
+ self.name = name
38
+ self.model = self.models[name]
39
+ return
40
  if name == "HED":
41
  self.model = HEDdetector.from_pretrained(self.MODEL_ID)
42
  elif name == "Midas":
 
61
  self.model = DepthEstimator()
62
  elif name == "UPerNet":
63
  self.model = ImageSegmentor()
64
+ elif name == "OneFormer":
65
+ self.model = ImageSegmentorOneFormer()
66
  else:
67
  raise ValueError
68
+ # if torch.cuda.is_available():
69
+ # torch.cuda.empty_cache()
70
+ # gc.collect()
71
  self.name = name
72
+ self.models[name] = self.model
73
 
74
  def __call__(self, image: PIL.Image.Image, **kwargs) -> PIL.Image.Image:
75
  if self.name == "Canny":
 
87
  image = HWC3(image)
88
  image = resize_image(image, resolution=detect_resolution)
89
  image = self.model(image, **kwargs)
90
+ if isinstance(image, tuple):
91
+ image = image[-1][...,::-1] # normal old
92
  image = HWC3(image)
93
  image = resize_image(image, resolution=image_resolution)
94
  return PIL.Image.fromarray(image)
settings.py CHANGED
@@ -7,7 +7,7 @@ DEFAULT_MODEL_ID = os.getenv("DEFAULT_MODEL_ID", "SG161222/Realistic_Vision_V4.0
7
  MAX_NUM_IMAGES = int(os.getenv("MAX_NUM_IMAGES", "3"))
8
  DEFAULT_NUM_IMAGES = min(MAX_NUM_IMAGES, int(os.getenv("DEFAULT_NUM_IMAGES", "3")))
9
  MAX_IMAGE_RESOLUTION = int(os.getenv("MAX_IMAGE_RESOLUTION", "768"))
10
- DEFAULT_IMAGE_RESOLUTION = min(MAX_IMAGE_RESOLUTION, int(os.getenv("DEFAULT_IMAGE_RESOLUTION", "768")))
11
 
12
  ALLOW_CHANGING_BASE_MODEL = os.getenv("SPACE_ID") != "HighCWu/control-lora-v3"
13
  SHOW_DUPLICATE_BUTTON = os.getenv("SHOW_DUPLICATE_BUTTON") == "1"
 
7
  MAX_NUM_IMAGES = int(os.getenv("MAX_NUM_IMAGES", "3"))
8
  DEFAULT_NUM_IMAGES = min(MAX_NUM_IMAGES, int(os.getenv("DEFAULT_NUM_IMAGES", "3")))
9
  MAX_IMAGE_RESOLUTION = int(os.getenv("MAX_IMAGE_RESOLUTION", "768"))
10
+ DEFAULT_IMAGE_RESOLUTION = min(MAX_IMAGE_RESOLUTION, int(os.getenv("DEFAULT_IMAGE_RESOLUTION", "512")))
11
 
12
  ALLOW_CHANGING_BASE_MODEL = os.getenv("SPACE_ID") != "HighCWu/control-lora-v3"
13
  SHOW_DUPLICATE_BUTTON = os.getenv("SHOW_DUPLICATE_BUTTON") == "1"
unet.py CHANGED
@@ -147,6 +147,8 @@ class UNet2DConditionModelEx(UNet2DConditionModel):
147
 
148
  def activate_extra_condition_adapters(self):
149
  lora_layers = [layer for layer in self.modules() if isinstance(layer, LoraLayer)]
 
 
150
  for lora_layer in lora_layers:
151
  adapter_names = [k for k in lora_layer.scaling.keys() if k in self.config.extra_condition_names]
152
  adapter_names += lora_layer.active_adapters
 
147
 
148
  def activate_extra_condition_adapters(self):
149
  lora_layers = [layer for layer in self.modules() if isinstance(layer, LoraLayer)]
150
+ if len(lora_layers) > 0:
151
+ self._hf_peft_config_loaded = True
152
  for lora_layer in lora_layers:
153
  adapter_names = [k for k in lora_layer.scaling.keys() if k in self.config.extra_condition_names]
154
  adapter_names += lora_layer.active_adapters