set_adapters performance degrades with the number of inactive adapters

[hrazjan]

Describe the bug

Goal

Build an image-generation service with StableDiffusionXLPipeline that:

1. Keeps ~50 LoRA adapters resident in GPU VRAM.
2. For each request:
   • activate ≤ 5 specific LoRAs via pipeline.set_adapters(...)
   • run inference
   • deactivate them (ready for the next request).

Issue

pipeline.set_adapters() becomes progressively slower the more unique LoRAs have ever been loaded,
even though each call still enables only up to five adapters.

# LoRAs ever loaded	set_adapters() time (s)
3	~ 0.1031
6	~ 0.1843
9	~ 0.2614
12	~ 0.3522
45	~ 1.2470
57	~ 1.5435

What I’ve tried

1. Load LoRAs from disk for every request ~ 0.8 s/LoRA, too slow.
2. Keep LoRAs in RAM (SpooledTemporaryFile) + pipeline.delete_adapter() – roughly as slow as (1).
3. Keep all 50 LoRAs on the GPU and just switch with set_adapters() – fastest so far, but still shows the O(N)-style growth above.

Question

Is this increasing latency expected?
Is there a recommended pattern for caching many LoRAs on the GPU and switching between small subsets without paying an O(total LoRAs) cost every time?

Any guidance (or confirmation it’s a current limitation) would be greatly appreciated!

Reproduction

Code

import os
import time
from typing import List
from pydantic import BaseModel
from diffusers import StableDiffusionXLPipeline, AutoencoderTiny
import torch
from diffusers.utils import logging
logging.disable_progress_bar()
logging.set_verbosity_error() 

pipeline = None

class Lora(BaseModel):
    name: str
    strength: float

def timeit(func):
    def wrapper(*args, **kwargs):
        start = time.time()
        result = func(*args, **kwargs)
        end = time.time()
        duration = end - start
        print(f"{func.__name__} executed in {duration:.4f} seconds")
        return result
    return wrapper

@timeit
def load_model():
    pipeline = StableDiffusionXLPipeline.from_pretrained(
        "stabilityai/stable-diffusion-xl-base-1.0",
        torch_dtype=torch.float16,
        vae=AutoencoderTiny.from_pretrained(
            'madebyollin/taesdxl',
            use_safetensors=True,
            torch_dtype=torch.float16,
        )
    ).to("cuda")
    pipeline.set_progress_bar_config(disable=True)

    return pipeline

@timeit
def set_adapters(pipeline, adapter_names, adapter_weights):
    pipeline.set_adapters(
        adapter_names=adapter_names,
        adapter_weights=adapter_weights,
    )

@timeit
def fuse_lora(pipeline):
    pipeline.fuse_lora()

@timeit
def inference(pipeline, req, generator=None):
    return pipeline(
        prompt=req.prompt,
        negative_prompt=req.negative_prompt,
        width=req.width,
        height=req.height,
        num_inference_steps=req.steps,
        guidance_scale=req.guidance_scale,
        generator=generator,
    ).images

def apply_loras(pipeline, loras: list[Lora]) -> str:
    if not loras or len(loras) == 0:
        pipeline.disable_lora()
        return
    
    pipeline.enable_lora()
    for lora in loras:
        try:
            pipeline.load_lora_weights(
                "ostris/super-cereal-sdxl-lora",
                weight_name="cereal_box_sdxl_v1.safetensors",
                adapter_name=lora.name,
                token=os.getenv("HUGGINGFACE_HUB_TOKEN", None),
            )
        except ValueError:
            continue # LoRA already loaded, skip
        except Exception as e:
            print(f"Failed to load LoRA {lora}: {e}")
            continue
    set_adapters(
        pipeline,
        adapter_names=[lora.name for lora in loras],
        adapter_weights=[lora.strength for lora in loras],
    )
    fuse_lora(pipeline)
    

    return

def generate_images(req, pipeline):
    generator = torch.Generator(device="cuda").manual_seed(42)

    apply_loras(pipeline, req.loras)

    images = inference(
        pipeline,
        req,
        generator=generator,
    )

    pipeline.unfuse_lora()
    
    return images

class GenerationRequest(BaseModel):
    prompt: str
    loras: List[Lora] = []
    negative_prompt: str = ""
    width: int = 512
    height: int = 512
    steps: int = 30
    guidance_scale: float = 7

def test_lora_group(pipeline, lora_group: List[Lora], group_number: int):    
    test_req = GenerationRequest(
        prompt="a simple test image",
        loras=[Lora(name=lora_name, strength=0.8) for lora_name in lora_group],
        width=256,
        height=256,
        steps=10,
    )
    
    try:
        generate_images(test_req, pipeline)
        return True, lora_group
    except Exception as e:
        return False, lora_group

def chunk_loras(lora_list: List[Lora], chunk_size: int = 5) -> List[List[Lora]]:
    """Split LoRAs into groups of specified size"""
    chunks = []
    for i in range(0, len(lora_list), chunk_size):
        chunks.append(lora_list[i:i + chunk_size])
    return chunks

def test():
    global pipeline
    # Load the pipeline if not already loaded
    if not pipeline:
        print("Loading pipeline for LoRA tests...")
        pipeline = load_model()
    
    # Split LoRAs into groups of 3
    all_loras = [f"cereal_{i}" for i in range(1, 50)]  # Example LoRAs
    lora_groups = chunk_loras(all_loras, 5)
    
    successful_groups = 0
    failed_loras = []
    
    print(f"Testing {len(all_loras)} LoRAs in {len(lora_groups)} groups of up to 5...")
    
    for i, lora_group in enumerate(lora_groups, 1):
        print(f"Testing group {i}/{len(lora_groups)}: {lora_group}")
        success, group = test_lora_group(pipeline, lora_group, i)
        if success:
            successful_groups += 1
        else:
            failed_loras.extend(group)
    
    if failed_loras:
        failed_names = [lora.value for lora in failed_loras]
        print(f"✗ Failed LoRAs: {', '.join(failed_names)}")

def test_all_loras():
    test()
    # run second test when all loras are loaded
    test()
    

if __name__ == "__main__":
    test_all_loras()

Logs

Above example gave me following results on RTX5090

Loading pipeline for LoRA tests...
load_model executed in 3.3787 seconds
Testing 49 LoRAs in 10 groups of up to 5...
Testing group 1/10: ['cereal_1', 'cereal_2', 'cereal_3', 'cereal_4', 'cereal_5']
/aibabe-diffusers/worker-env/lib/python3.11/site-packages/peft/tuners/tuners_utils.py:167: UserWarning: Already found a `peft_config` attribute in the model. This will lead to having multiple adapters in the model. Make sure to know what you are doing!
  warnings.warn(
set_adapters executed in 0.1883 seconds
fuse_lora executed in 0.1787 seconds
inference executed in 0.4306 seconds
Testing group 2/10: ['cereal_6', 'cereal_7', 'cereal_8', 'cereal_9', 'cereal_10']
set_adapters executed in 0.6675 seconds
fuse_lora executed in 0.1478 seconds
inference executed in 0.2558 seconds
Testing group 3/10: ['cereal_11', 'cereal_12', 'cereal_13', 'cereal_14', 'cereal_15']
set_adapters executed in 0.6900 seconds
fuse_lora executed in 0.2076 seconds
inference executed in 0.3285 seconds
Testing group 4/10: ['cereal_16', 'cereal_17', 'cereal_18', 'cereal_19', 'cereal_20']
set_adapters executed in 0.7407 seconds
fuse_lora executed in 0.1937 seconds
inference executed in 0.2802 seconds
Testing group 5/10: ['cereal_21', 'cereal_22', 'cereal_23', 'cereal_24', 'cereal_25']
set_adapters executed in 0.9166 seconds
fuse_lora executed in 0.2169 seconds
inference executed in 0.2973 seconds
Testing group 6/10: ['cereal_26', 'cereal_27', 'cereal_28', 'cereal_29', 'cereal_30']
set_adapters executed in 1.7071 seconds
fuse_lora executed in 0.2461 seconds
inference executed in 0.3163 seconds
Testing group 7/10: ['cereal_31', 'cereal_32', 'cereal_33', 'cereal_34', 'cereal_35']
set_adapters executed in 1.8331 seconds
fuse_lora executed in 0.2640 seconds
inference executed in 0.3335 seconds
Testing group 8/10: ['cereal_36', 'cereal_37', 'cereal_38', 'cereal_39', 'cereal_40']
set_adapters executed in 2.0735 seconds
fuse_lora executed in 0.2882 seconds
inference executed in 0.3425 seconds
Testing group 9/10: ['cereal_41', 'cereal_42', 'cereal_43', 'cereal_44', 'cereal_45']
set_adapters executed in 2.3036 seconds
fuse_lora executed in 0.3111 seconds
inference executed in 0.3606 seconds
Testing group 10/10: ['cereal_46', 'cereal_47', 'cereal_48', 'cereal_49']
set_adapters executed in 2.2296 seconds
fuse_lora executed in 0.3228 seconds
inference executed in 0.3789 seconds
Testing 49 LoRAs in 10 groups of up to 5...
Testing group 1/10: ['cereal_1', 'cereal_2', 'cereal_3', 'cereal_4', 'cereal_5']
set_adapters executed in 1.7317 seconds
fuse_lora executed in 0.3396 seconds
inference executed in 0.3807 seconds
Testing group 2/10: ['cereal_6', 'cereal_7', 'cereal_8', 'cereal_9', 'cereal_10']
set_adapters executed in 2.5145 seconds
fuse_lora executed in 0.3403 seconds
inference executed in 0.3808 seconds
Testing group 3/10: ['cereal_11', 'cereal_12', 'cereal_13', 'cereal_14', 'cereal_15']
set_adapters executed in 1.7341 seconds
fuse_lora executed in 0.3392 seconds
inference executed in 0.3820 seconds
Testing group 4/10: ['cereal_16', 'cereal_17', 'cereal_18', 'cereal_19', 'cereal_20']
set_adapters executed in 2.5293 seconds
fuse_lora executed in 0.3392 seconds
inference executed in 0.3761 seconds
Testing group 5/10: ['cereal_21', 'cereal_22', 'cereal_23', 'cereal_24', 'cereal_25']
set_adapters executed in 1.7418 seconds
fuse_lora executed in 0.3438 seconds
inference executed in 0.3790 seconds
Testing group 6/10: ['cereal_26', 'cereal_27', 'cereal_28', 'cereal_29', 'cereal_30']
set_adapters executed in 2.5821 seconds
fuse_lora executed in 0.3522 seconds
inference executed in 0.3876 seconds
Testing group 7/10: ['cereal_31', 'cereal_32', 'cereal_33', 'cereal_34', 'cereal_35']
set_adapters executed in 2.5645 seconds
fuse_lora executed in 0.3373 seconds
inference executed in 0.3804 seconds
Testing group 8/10: ['cereal_36', 'cereal_37', 'cereal_38', 'cereal_39', 'cereal_40']
set_adapters executed in 1.8011 seconds
fuse_lora executed in 0.3375 seconds
inference executed in 0.3843 seconds
Testing group 9/10: ['cereal_41', 'cereal_42', 'cereal_43', 'cereal_44', 'cereal_45']
set_adapters executed in 2.5058 seconds
fuse_lora executed in 0.3371 seconds
inference executed in 0.3810 seconds
Testing group 10/10: ['cereal_46', 'cereal_47', 'cereal_48', 'cereal_49']
set_adapters executed in 1.4389 seconds
fuse_lora executed in 0.3215 seconds
inference executed in 0.3849 seconds

System Info

• 🤗 Diffusers version: 0.34.0
• Platform: Linux-6.8.0-58-generic-x86_64-with-glibc2.35
• Running on Google Colab?: No
• Python version: 3.11.11
• PyTorch version (GPU?): 2.7.1+cu128 (True)
• Flax version (CPU?/GPU?/TPU?): not installed (NA)
• Jax version: not installed
• JaxLib version: not installed
• Huggingface_hub version: 0.33.1
• Transformers version: 4.53.0
• Accelerate version: 1.8.1
• PEFT version: 0.15.2
• Bitsandbytes version: not installed
• Safetensors version: 0.5.3
• xFormers version: not installed
• Accelerator: NVIDIA GeForce RTX 5090, 32607 MiB
• Using GPU in script?: yes
• Using distributed or parallel set-up in script?: no

Who can help?

@sayakpaul

[sayakpaul]

Cc: @BenjaminBossan too.

Maybe we could keep a small number of LoRAs on RAM by calling

diffusers/src/diffusers/loaders/lora_base.py

Line 919 in d7dd924

def set_lora_device(self, adapter_names: List[str], device: Union[torch.device, str, int]) -> None:

?

Additionally, if you could provide us with a minimal reproducible snippet so that we could obtain the timings that would be helpful. Sometimes, hardware, disk bandwidth, etc. do creep into these things.

[BenjaminBossan]

I think we first need to identify where the time is spent before jumping to solutions. A small reproducer would be ideal for that.

[hrazjan]

Hi @sayakpaul and @BenjaminBossan, I've updated the bug description with a better example and logs it produced on my setup. Let me know it this is good enough. Thanks for looking into this.

[BenjaminBossan]

Thanks for that @hrazjan. Here are my results:

set_adapters executed in 0.5061 seconds
fuse_lora executed in 0.2195 seconds
inference executed in 0.5384 seconds
Testing group 2/10: ['cereal_6', 'cereal_7', 'cereal_8', 'cereal_9', 'cereal_10']
set_adapters executed in 0.4466 seconds
fuse_lora executed in 0.1894 seconds
inference executed in 0.3834 seconds
Testing group 3/10: ['cereal_11', 'cereal_12', 'cereal_13', 'cereal_14', 'cereal_15']
set_adapters executed in 0.6302 seconds
fuse_lora executed in 0.2163 seconds
inference executed in 0.4036 seconds
Testing group 4/10: ['cereal_16', 'cereal_17', 'cereal_18', 'cereal_19', 'cereal_20']
set_adapters executed in 0.8395 seconds
fuse_lora executed in 0.2492 seconds
inference executed in 0.4246 seconds
Testing group 5/10: ['cereal_21', 'cereal_22', 'cereal_23', 'cereal_24', 'cereal_25']
set_adapters executed in 1.4510 seconds
fuse_lora executed in 0.2683 seconds
inference executed in 0.4354 seconds
Testing group 6/10: ['cereal_26', 'cereal_27', 'cereal_28', 'cereal_29', 'cereal_30']
set_adapters executed in 1.6897 seconds
fuse_lora executed in 0.2958 seconds
inference executed in 0.4524 seconds
Testing group 7/10: ['cereal_31', 'cereal_32', 'cereal_33', 'cereal_34', 'cereal_35']
set_adapters executed in 1.4007 seconds
fuse_lora executed in 0.3228 seconds
inference executed in 0.4780 seconds
Testing group 8/10: ['cereal_36', 'cereal_37', 'cereal_38', 'cereal_39', 'cereal_40']
set_adapters executed in 1.6028 seconds
fuse_lora executed in 0.3525 seconds
inference executed in 0.4877 seconds
Testing group 9/10: ['cereal_41', 'cereal_42', 'cereal_43', 'cereal_44', 'cereal_45']
set_adapters executed in 1.8075 seconds
fuse_lora executed in 0.3738 seconds
inference executed in 0.5127 seconds
Testing group 10/10: ['cereal_46', 'cereal_47', 'cereal_48', 'cereal_49']
set_adapters executed in 2.2938 seconds
fuse_lora executed in 0.3811 seconds
inference executed in 0.5186 seconds
Testing 49 LoRAs in 10 groups of up to 5...
Testing group 1/10: ['cereal_1', 'cereal_2', 'cereal_3', 'cereal_4', 'cereal_5']
set_adapters executed in 1.9787 seconds
fuse_lora executed in 0.4013 seconds
inference executed in 0.5190 seconds
Testing group 2/10: ['cereal_6', 'cereal_7', 'cereal_8', 'cereal_9', 'cereal_10']
set_adapters executed in 2.6414 seconds
fuse_lora executed in 0.4009 seconds
inference executed in 0.5124 seconds
Testing group 3/10: ['cereal_11', 'cereal_12', 'cereal_13', 'cereal_14', 'cereal_15']
set_adapters executed in 1.9644 seconds
fuse_lora executed in 0.3925 seconds
inference executed in 0.5151 seconds
Testing group 4/10: ['cereal_16', 'cereal_17', 'cereal_18', 'cereal_19', 'cereal_20']
set_adapters executed in 2.6315 seconds
fuse_lora executed in 0.3968 seconds
inference executed in 0.5186 seconds
Testing group 5/10: ['cereal_21', 'cereal_22', 'cereal_23', 'cereal_24', 'cereal_25']
set_adapters executed in 1.9647 seconds
fuse_lora executed in 0.4023 seconds
inference executed in 0.5193 seconds
Testing group 6/10: ['cereal_26', 'cereal_27', 'cereal_28', 'cereal_29', 'cereal_30']
set_adapters executed in 2.6223 seconds
fuse_lora executed in 0.3962 seconds
inference executed in 0.5048 seconds
Testing group 7/10: ['cereal_31', 'cereal_32', 'cereal_33', 'cereal_34', 'cereal_35']
set_adapters executed in 1.9642 seconds
fuse_lora executed in 0.4016 seconds
inference executed in 0.5168 seconds
Testing group 8/10: ['cereal_36', 'cereal_37', 'cereal_38', 'cereal_39', 'cereal_40']
set_adapters executed in 2.6285 seconds
fuse_lora executed in 0.3991 seconds
inference executed in 0.5124 seconds
Testing group 9/10: ['cereal_41', 'cereal_42', 'cereal_43', 'cereal_44', 'cereal_45']
set_adapters executed in 1.9712 seconds
fuse_lora executed in 0.3981 seconds
inference executed in 0.5147 seconds
Testing group 10/10: ['cereal_46', 'cereal_47', 'cereal_48', 'cereal_49']
set_adapters executed in 1.6291 seconds
fuse_lora executed in 0.3822 seconds
inference executed in 0.5167 seconds

Is this consistent with what you got? And to be clear, the issue is the increase in fuse_lora time that we see in the first run as we progressively load more LoRAs, right?

[sayakpaul]

I think it's the set_adapters() method. fuse_lora() timing seems expected to me.

[hrazjan]

Yes, it's consistent. The fuse_lora is increasing, but only slightly. The main problem is set_adapters. As you can see the first run takes only 0.1883 seconds, but the last ones can take up to 2.6285 seconds

Alternative solution that came to my mind is something like this:

# preload all state_dicts to GPU
def create_lora_registry(pipeline: StableDiffusionXLPipeline):
    for name, path in lora_paths.items():
        if name in lora_registry:
            continue
        lora_registry[name], _ = pipeline.lora_state_dict(path)
create_lora_registry(pipeline)

...
## generate image
#load lora from dict
entry = lora_registry[lora.name]
pipeline.load_lora_weights(
    pretrained_model_name_or_path_or_dict=entry,
    adapter_name=lora.name,
)
# set_adapters
# inference
# delete_adapters

This would keep the loras in VRAM (i think), but the number of total adapters would be small and set_adapters should be fast. Unfortunately I wasn't able to make it run.

Traceback (most recent call last):
  File "/aibabe-diffusers/aibabe_diffusers/main.py", line 89, in load_loras
    pipeline.load_lora_weights(
  File "/aibabe-diffusers/worker-env/lib/python3.11/site-packages/diffusers/loaders/lora_pipeline.py", line 686, in load_lora_weights
    self.load_lora_into_unet(
  File "/aibabe-diffusers/worker-env/lib/python3.11/site-packages/diffusers/loaders/lora_pipeline.py", line 889, in load_lora_into_unet
    unet.load_lora_adapter(
  File "/aibabe-diffusers/worker-env/lib/python3.11/site-packages/diffusers/loaders/peft.py", line 311, in load_lora_adapter
    inject_adapter_in_model(lora_config, self, adapter_name=adapter_name, **peft_kwargs)
  File "/aibabe-diffusers/worker-env/lib/python3.11/site-packages/peft/mapping.py", line 76, in inject_adapter_in_model
    peft_model = tuner_cls(model, peft_config, adapter_name=adapter_name, low_cpu_mem_usage=low_cpu_mem_usage)
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/aibabe-diffusers/worker-env/lib/python3.11/site-packages/peft/tuners/lora/model.py", line 142, in __init__
    super().__init__(model, config, adapter_name, low_cpu_mem_usage=low_cpu_mem_usage)
  File "/aibabe-diffusers/worker-env/lib/python3.11/site-packages/peft/tuners/tuners_utils.py", line 180, in __init__
    self.inject_adapter(self.model, adapter_name, low_cpu_mem_usage=low_cpu_mem_usage)
  File "/aibabe-diffusers/worker-env/lib/python3.11/site-packages/peft/tuners/tuners_utils.py", line 527, in inject_adapter
    raise ValueError(error_msg)
ValueError: Target modules {'0.1.transformer_blocks.0.attn1.to_v', '7.1.transformer_blocks.0.attn2.to_out.0', ... } not found in the base model. Please check the target modules and try again.

[BenjaminBossan]

I did some quick profiling, if line_profiler is not lying to me, the majority of time (95% of time for 50 LoRAs) is spent on here:

diffusers/src/diffusers/loaders/peft.py

Line 482 in cdaf84a

weights = scale_expansion_fn(self, weights)

There, the majority comes from:

diffusers/src/diffusers/loaders/unet_loader_utils.py

Line 55 in cdaf84a

unet.state_dict(),

Probably there is room for improvement here.

@hrazjan If you comment out the weights = scale_expansion_fn(self, weights) line, do you see improved results?

[BenjaminBossan]

I re-ran the benchmark script with weight scaling commented out. There is still an increase in the set_adapters time, but overall it's 10x faster, thus overall, this should not be a bottle neck.

set_adapters executed in 0.0373 seconds
fuse_lora executed in 0.2179 seconds
inference executed in 0.5062 seconds
Testing group 2/10: ['cereal_6', 'cereal_7', 'cereal_8', 'cereal_9', 'cereal_10']
set_adapters executed in 0.0628 seconds
fuse_lora executed in 0.1841 seconds
inference executed in 0.3688 seconds
Testing group 3/10: ['cereal_11', 'cereal_12', 'cereal_13', 'cereal_14', 'cereal_15']
set_adapters executed in 0.0900 seconds
fuse_lora executed in 0.2104 seconds
inference executed in 0.3897 seconds
Testing group 4/10: ['cereal_16', 'cereal_17', 'cereal_18', 'cereal_19', 'cereal_20']
set_adapters executed in 0.1173 seconds
fuse_lora executed in 0.2417 seconds
inference executed in 0.4012 seconds
Testing group 5/10: ['cereal_21', 'cereal_22', 'cereal_23', 'cereal_24', 'cereal_25']
set_adapters executed in 0.1441 seconds
fuse_lora executed in 0.2650 seconds
inference executed in 0.4212 seconds
Testing group 6/10: ['cereal_26', 'cereal_27', 'cereal_28', 'cereal_29', 'cereal_30']
set_adapters executed in 0.1750 seconds
fuse_lora executed in 0.2900 seconds
inference executed in 0.4554 seconds
Testing group 7/10: ['cereal_31', 'cereal_32', 'cereal_33', 'cereal_34', 'cereal_35']
set_adapters executed in 0.2063 seconds
fuse_lora executed in 0.3178 seconds
inference executed in 0.4499 seconds
Testing group 8/10: ['cereal_36', 'cereal_37', 'cereal_38', 'cereal_39', 'cereal_40']
set_adapters executed in 0.2324 seconds
fuse_lora executed in 0.3466 seconds
inference executed in 0.4743 seconds
Testing group 9/10: ['cereal_41', 'cereal_42', 'cereal_43', 'cereal_44', 'cereal_45']
set_adapters executed in 0.2589 seconds
fuse_lora executed in 0.3733 seconds
inference executed in 0.4896 seconds
Testing group 10/10: ['cereal_46', 'cereal_47', 'cereal_48', 'cereal_49']
set_adapters executed in 0.2826 seconds
fuse_lora executed in 0.3714 seconds
inference executed in 0.5036 seconds
Testing 49 LoRAs in 10 groups of up to 5...
Testing group 1/10: ['cereal_1', 'cereal_2', 'cereal_3', 'cereal_4', 'cereal_5']
set_adapters executed in 0.2788 seconds
fuse_lora executed in 0.3942 seconds
inference executed in 0.4996 seconds
Testing group 2/10: ['cereal_6', 'cereal_7', 'cereal_8', 'cereal_9', 'cereal_10']
set_adapters executed in 0.2786 seconds
fuse_lora executed in 0.3940 seconds
inference executed in 0.5099 seconds
Testing group 3/10: ['cereal_11', 'cereal_12', 'cereal_13', 'cereal_14', 'cereal_15']
set_adapters executed in 0.2814 seconds
fuse_lora executed in 0.3980 seconds
inference executed in 0.5006 seconds
Testing group 4/10: ['cereal_16', 'cereal_17', 'cereal_18', 'cereal_19', 'cereal_20']
set_adapters executed in 0.2784 seconds
fuse_lora executed in 0.3960 seconds
inference executed in 0.5058 seconds
Testing group 5/10: ['cereal_21', 'cereal_22', 'cereal_23', 'cereal_24', 'cereal_25']
set_adapters executed in 0.2825 seconds
fuse_lora executed in 0.4038 seconds
inference executed in 0.5029 seconds
Testing group 6/10: ['cereal_26', 'cereal_27', 'cereal_28', 'cereal_29', 'cereal_30']
set_adapters executed in 0.2840 seconds
fuse_lora executed in 0.3912 seconds
inference executed in 0.4997 seconds
Testing group 7/10: ['cereal_31', 'cereal_32', 'cereal_33', 'cereal_34', 'cereal_35']
set_adapters executed in 0.2862 seconds
fuse_lora executed in 0.4066 seconds
inference executed in 0.5199 seconds
Testing group 8/10: ['cereal_36', 'cereal_37', 'cereal_38', 'cereal_39', 'cereal_40']
set_adapters executed in 0.2856 seconds
fuse_lora executed in 0.4041 seconds
inference executed in 0.5159 seconds
Testing group 9/10: ['cereal_41', 'cereal_42', 'cereal_43', 'cereal_44', 'cereal_45']
set_adapters executed in 0.2786 seconds
fuse_lora executed in 0.3998 seconds
inference executed in 0.5047 seconds
Testing group 10/10: ['cereal_46', 'cereal_47', 'cereal_48', 'cereal_49']
set_adapters executed in 0.2794 seconds
fuse_lora executed in 0.3740 seconds
inference executed in 0.5033 seconds

[hrazjan]

@BenjaminBossan I commented out that line and the results match yours. Thanks a lot. I will try this on my whole project. Hopefully, it should not have any effect on the image quality. I tried with debugger and the scale_expansion_fn just returned whatever I set in adapter_weights.

[BenjaminBossan]

I'm sure we can optimize the function to make it fast without outright skipping it :) But yeah, for the time being, please test this workaround and let us know if it helps.

[sayakpaul]

I can look into optimizing it but for the scale expansion stuff please refer to:
https://huggingface.co/docs/diffusers/main/en/tutorials/using_peft_for_inference?weight-scale=finer+control#hotswapping

Since in this case we're not supplying any custom scaling the scale_expansion_fn() just returns the weights set through adapter_weights().

[BenjaminBossan]

There is a very simple optimization, which is to only call unet.state_dict() if necessary instead of calling it early, as it's expensive and often not needed. The patch is:

index 274665204..1856292fd 100644
--- a/src/diffusers/loaders/unet_loader_utils.py
+++ b/src/diffusers/loaders/unet_loader_utils.py
@@ -14,6 +14,8 @@
 import copy
 from typing import TYPE_CHECKING, Dict, List, Union
 
+from torch import nn
+
 from ..utils import logging
 
 
@@ -52,7 +54,7 @@ def _maybe_expand_lora_scales(
             weight_for_adapter,
             blocks_with_transformer,
             transformer_per_block,
-            unet.state_dict(),
+            model=unet,
             default_scale=default_scale,
         )
         for weight_for_adapter in weight_scales
@@ -65,7 +67,7 @@ def _maybe_expand_lora_scales_for_one_adapter(
     scales: Union[float, Dict],
     blocks_with_transformer: Dict[str, int],
     transformer_per_block: Dict[str, int],
-    state_dict: None,
+    model: nn.Module,
     default_scale: float = 1.0,
 ):
     """
@@ -155,7 +157,7 @@ def _maybe_expand_lora_scales_for_one_adapter(
         del scales[updown]
 
     for layer in scales.keys():
-        if not any(_translate_into_actual_layer_name(layer) in module for module in state_dict.keys()):
+        if not any(_translate_into_actual_layer_name(layer) in module for module in model.state_dict().keys()):
             raise ValueError(
                 f"Can't set lora scale for layer {layer}. It either doesn't exist in this unet or it has no attentions."
             )

As in this case, we early return from _maybe_expand_lora_scales_for_one_adapter, the state_dict() call is never performed, keeping the code quick. When I re-ran the script, I got the roughly same results as above with the code commented out.

Also, iterating through model.named_modules() should be faster than model.state_dict(), at least in my testing that's what I found. As we only need the module names, we could change that too.

@sayakpaul WDYT?

[sayakpaul]

@BenjaminBossan thanks a lot for looking into this! Would you be able to make a PR for this and we can take it from there?