update

app.py

@@ -5,7 +5,7 @@ import cv2
 import gradio as gr
 import numpy as np
 import PIL
-import spaces
+#import spaces
 import torch
 from diffusers.models import ControlNetModel
 from diffusers.utils import load_image
@@ -17,7 +17,16 @@ from style_template import styles
 
 # global variable
 MAX_SEED = np.iinfo(np.int32).max
-device = "cuda" if torch.cuda.is_available() else "cpu"
+#device = "cuda" if torch.cuda.is_available() else "cpu"
+torch_dtype = torch.float16
+if torch.backends.mps.is_available():
+    device = "mps"
+    torch_dtype = torch.float32
+elif torch.cuda.is_available():
+    device = "cuda"
+else:
+    device = "cpu"
+
 STYLE_NAMES = list(styles.keys())
 DEFAULT_STYLE_NAME = "Watercolor"
 
@@ -31,6 +40,7 @@ hf_hub_download(
     local_dir="./checkpoints",
 )
 hf_hub_download(repo_id="InstantX/InstantID", filename="ip-adapter.bin", local_dir="./checkpoints")
+hf_hub_download(repo_id="latent-consistency/lcm-lora-sdxl", filename="pytorch_lora_weights.safetensors", local_dir="./checkpoints")
 
 # Load face encoder
 app = FaceAnalysis(name="antelopev2", root="./", providers=["CPUExecutionProvider"])
@@ -39,23 +49,49 @@ app.prepare(ctx_id=0, det_size=(640, 640))
 # Path to InstantID models
 face_adapter = "./checkpoints/ip-adapter.bin"
 controlnet_path = "./checkpoints/ControlNetModel"
+lcm_lora_path = "./checkpoints/pytorch_lora_weights.safetensors"
 
 # Load pipeline
-controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=torch.float16)
+#controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=torch.float16)
+controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=torch_dtype)
 
 base_model_path = "wangqixun/YamerMIX_v8"
 
 pipe = StableDiffusionXLInstantIDPipeline.from_pretrained(
     base_model_path,
     controlnet=controlnet,
-    torch_dtype=torch.float16,
+    #torch_dtype=torch.float16,
+    torch_dtype=torch_dtype,
     safety_checker=None,
     feature_extractor=None,
 )
-pipe.cuda()
+#pipe.cuda()
+
+num_inference_steps = 30
+guidance_scale = 5
+
++# LCM
+if os.environ.get("LCM"):
+    num_inference_steps = 10
+    guidance_scale = 0
+
+    pipe.load_lora_weights(lcm_lora_path)
+    pipe.fuse_lora()
+    pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config)
+
+if device == 'mps':
+    pipe.to("mps", torch_dtype)
+    pipe.enable_attention_slicing()
+elif device == 'cuda':
+    pipe.cuda()
+
 pipe.load_ip_adapter_instantid(face_adapter)
-pipe.image_proj_model.to("cuda")
-pipe.unet.to("cuda")
+#pipe.image_proj_model.to("cuda")
+#pipe.unet.to("cuda")
+if device == 'mps' or device == 'cuda':
+    pipe.image_proj_model.to(device)
+    pipe.unet.to(device)
+
 
 
 def randomize_seed_fn(seed: int, randomize_seed: bool) -> int:
@@ -187,7 +223,7 @@ def check_input_image(face_image):
         raise gr.Error("Cannot find any input face image! Please upload the face image")
 
 
-@spaces.GPU
+#@spaces.GPU
 def generate_image(
     face_image_path,
     pose_image_path,
@@ -369,14 +405,14 @@ with gr.Blocks(css=css) as demo:
                     minimum=20,
                     maximum=100,
                     step=1,
-                    value=30,
+                    value=num_inference_steps,
                 )
                 guidance_scale = gr.Slider(
                     label="Guidance scale",
                     minimum=0.1,
                     maximum=10.0,
                     step=0.1,
-                    value=5,
+                    value=guidance_scale,
                 )
                 seed = gr.Slider(
                     label="Seed",

requirements.txt

@@ -1,14 +1,15 @@
 diffusers==0.25.0
-torch==2.0.0
-torchvision==0.15.1
+#torch==2.0.0
+#torchvision==0.15.1
 transformers==4.36.2
 accelerate
 safetensors
 einops
-onnxruntime-gpu
+#onnxruntime-gpu
+onnxruntime
 spaces==0.19.4
 omegaconf
 peft
 huggingface-hub==0.20.2
 opencv-python
-insightface
+insightface