update readme for v121

siliconflow · Mar 1, 2024 · 59b454e · 59b454e
2 parents 2942049 + d3a5629
commit 59b454e
Show file tree

Hide file tree

Showing 16 changed files with 176 additions and 568 deletions.
diff --git a/.github/workflows/pub.yml b/.github/workflows/pub.yml
@@ -22,8 +22,14 @@ jobs:
         id: date
         run: |
           sed -i '1d' "src/onediff/__init__.py"
+          sed -i '1d' "onediff_diffusers_extensions/onediffx/__init__.py"
           formatted_date=$(date +'%Y%m%d%H%M')
+<<<<<<< HEAD
           echo "__version__ = \"0.12.1${formatted_date}\"" >> src/onediff/__init__.py
+=======
+          echo "__version__ = \"0.12.1.dev${formatted_date}\"" >> src/onediff/__init__.py
+          echo "__version__ = \"0.12.1.dev${formatted_date}\"" >> onediff_diffusers_extensions/onediffx/__init__.py
+>>>>>>> d3a5629f515e3dda2f9c46ea9907668714f2f642
       - run: cat src/onediff/__init__.py
       - run: python3 -m pip install build
       - run: python3 -m build
@@ -32,3 +38,11 @@ jobs:
         with:
           user: __token__
           password: ${{ secrets.PYPI_API_TOKEN }}
+      - run: python3 -m build
+        working-directory: ./onediff_diffusers_extensions
+      - name: Publish extension
+        uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN_ONEDIFF_X }}
+          packages_dir: ./onediff_diffusers_extensions/dist
diff --git a/.github/workflows/sd.yml b/.github/workflows/sd.yml
@@ -5,6 +5,7 @@ on:
     branches:
       - main
       - enable-docker-hub-cache
+      - "img/*"
   schedule:
     # every 6 hours
     - cron: "0 */6 * * *"

diff --git a/Dockerfile b/Dockerfile
@@ -8,3 +8,4 @@ RUN pip install -f ${ONEFLOW_PIP_INDEX} ${ONEFLOW_PACKAGE_NAME}
 RUN python3 -m pip install "torch" "transformers==4.27.1" "diffusers[torch]==0.19.3"
 ADD . /src/onediff
 RUN python3 -m pip install -e /src/onediff
+RUN python3 -m pip install -e /src/onediff/onediff_diffusers_extensions
diff --git a/README.md b/README.md
@@ -7,61 +7,86 @@
 [![Docker image build](https://github.com/siliconflow/onediff/actions/workflows/sd.yml/badge.svg)](https://github.com/siliconflow/onediff/actions/workflows/sd.yml)
 [![Run examples](https://github.com/siliconflow/onediff/actions/workflows/examples.yml/badge.svg?event=schedule)](https://github.com/siliconflow/onediff/actions/workflows/examples.yml?query=event%3Aschedule)
 
-OneDiff is the abbreviation of "**one** line of code to accelerate **diff**usion models".
 
-OneDiff is a Python package that provides two high-level features:
+OneDiff provides an out-of-the-box acceleration for diffusion models:
 - PyTorch Module compilation tools and strong optimized GPU Kernels for diffusion models
 - Out-of-the-box acceleration for popular UIs/libs
   - [OneDiff for HF diffusers 🤗](https://github.com/siliconflow/onediff/tree/main/onediff_diffusers_extensions)
   - [OneDiff for ComfyUI](https://github.com/siliconflow/onediff/tree/main/onediff_comfy_nodes)
   - [OneDiff for Stable Diffusion web UI](https://github.com/siliconflow/onediff/tree/main/onediff_sd_webui_extensions)
 
-## News
+OneDiff is the abbreviation of "**one** line of code to accelerate **diff**usion models". Here is the latested news:
+
 - :rocket:[Accelerating Stable Video Diffusion 3x faster with OneDiff DeepCache + Int8](https://www.reddit.com/r/StableDiffusion/comments/1adu2hn/accelerating_stable_video_diffusion_3x_faster/)
 - :rocket:[Accelerating SDXL 3x faster with DeepCache and OneDiff](https://www.reddit.com/r/StableDiffusion/comments/18lz2ir/accelerating_sdxl_3x_faster_with_deepcache_and/)
 - :rocket:[InstantID can run 1.8x Faster with OneDiff](https://www.reddit.com/r/StableDiffusion/comments/1al19ek/instantid_can_run_18x_faster_with_onediff/)
 
-## Roadmap
-[OneDiff Development Roadmap](https://github.com/siliconflow/onediff/wiki#onediff-roadmap)
-
-## Community & Support
-- [Create an issue](https://github.com/siliconflow/onediff/issues)
-- Chat in Discord: [![](https://dcbadge.vercel.app/api/server/RKJTjZMcPQ?style=plastic)](https://discord.gg/RKJTjZMcPQ)
-- Email for Enterprise Edition or other business inquiries: [email protected]
-
-## State-of-the-art performance
-
-<img src="benchmarks/pics/240105_sdxl_e2e_a100.png" width="600" height="400">
-
-## Easy to use
-#### Out-of-the-box acceleration for popular UIs/libs
-  - [HF diffusers 🤗](https://github.com/siliconflow/onediff/tree/main/onediff_diffusers_extensions)
-  - [ComfyUI](https://github.com/siliconflow/onediff/tree/main/onediff_comfy_nodes)
-  - [Stable Diffusion web UI](https://github.com/siliconflow/onediff/tree/main/onediff_sd_webui_extensions)
-#### Acceleration for state-of-the-art Models
-  - [SDXL](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_sdxl.py)
-  - [SDXL Turbo](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_sdxl_turbo.py)
-  - [SD 1.5/2.1](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image.py)
-  - [LoRA (and dynamic switching LoRA)](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_sdxl_lora.py)
-  - [ControlNet](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_controlnet.py)
-  - [LCM](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_lcm.py) and [LCM LoRA](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_lcm_lora_sdxl.py)
-  - [Stable Video Diffusion](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/image_to_video.py)
-  - [DeepCache](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_deep_cache_sdxl.py)
-  - [InstantID](https://github.com/siliconflow/onediff/blob/main/benchmarks/instant_id.py)
-
-#### Ready for production
-  - [Acceleration with oneflow_compile](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_sdxl.py)
-  - [Support Multi-resolution input](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_sdxl.py)
-  - Compile and save the compiled result offline, then load it online for serving
-    - [Save and Load the compiled graph](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_sdxl_save_load.py)
-    - [Change device of the compiled graph to do multi-process serving](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_sdxl_mp_load.py)
-
-
-  - [HF diffusers 🤗](https://github.com/siliconflow/onediff/tree/main/onediff_diffusers_extensions)
-  - [ComfyUI](https://github.com/siliconflow/onediff/tree/main/onediff_comfy_nodes)
-  - [Stable Diffusion web UI](https://github.com/siliconflow/onediff/tree/main/onediff_sd_webui_extensions)
-
-## OneDiff Enterprise Edition
+The Full introduction of OneDiff:
+<!-- toc -->
+- [More About OneDiff](#more-about-onediff)
+  - [State-of-the-art performance](#state-of-the-art-performance)
+  - [Acceleration for production](#acceleration-for-production)
+  - [Acceleration for State-of-the-art models](#acceleration-for-state-of-the-art-models)
+  - [OneDiff Enterprise Edition](#onediff-enterprise-edition)
+  - [Roadmap](#roadmap)
+- [Community and Support](#community-and-support)
+- [Installation](#installation)
+- [Release](#release)
+
+<!-- tocstop -->
+
+## More About OneDiff
+
+### State-of-the-art performance
+#### SDXL E2E time
+- Model stabilityai/stable-diffusion-xl-base-1.0;
+- Image size 1024*1024, batch size 1, steps 30;
+- NVIDIA A100 80G SXM4;
+
+<img src="imgs/0_12_sdxl.png" height="400">
+
+#### SVD E2E time
+- Model stabilityai/stable-video-diffusion-img2vid-xt;
+- Image size 576*1024, batch size 1, steps 25, decoder chunk size 5;
+- NVIDIA A100 80G SXM4;
+
+<img src="imgs/0_12_svd.png" height="400">
+
+### Acceleration for State-of-the-art models
+OneDiff support the acceleratioin for SOTA models.
+| AIGC Type | Models                      | HF diffusers |            | ComfyUI   |            | SD web UI |            |
+| --------- | --------------------------- | ------------ | ---------- | --------- | ---------- | --------- | ---------- |
+|           |                             | Community    | Enterprise | Community | Enterprise | Community | Enterprise |
+| Image     | SD 1.5                      | stable       | stable     | stable    | stable     | beta      | beta       |
+|           | SD 2.1                      | stable       | stable     | stable    | stable     | beta      | beta       |
+|           | SDXL                        | stable       | stable     | stable    | stable     | beta      | beta       |
+|           | LoRA                        | stable       |            | stable    |            | beta      |            |
+|           | ControlNet                  | stable       |            | stable    |            |           |            |
+|           | SDXL Turbo                  | stable       |            | stable    |            |           |            |
+|           | LCM                         | stable       |            | stable    |            |           |            |
+|           | SDXL DeepCache              | stable       | beta       | stable    | beta       |           |            |
+|           | InstantID                   | stable       |            | stable    |            |           |            |
+| Video     | SVD(stable Video Diffusion) | stable       | beta       | stable    | beta       |           |            |
+|           | SVD DeepCache               | stable       | beta       | stable    | beta       |           |            |
+
+**Note: Enterprise Edition contains all the functionality in Community Edition.**
+* stable: release for public usage, and has long-term support; 
+* beta: release for professional usage, and has long-term support; 
+* alpha: early release for expert usage, and is **under active development**; 
+
+
+
+### Acceleration for production
+#### PyTorch Module compilation
+- [compilation with oneflow_compile](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_sdxl.py)
+#### Avoid compilation time for new input shape
+- [Support Multi-resolution input](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_sdxl.py)
+#### Avoid compilation time for online serving
+Compile and save the compiled result offline, then load it online for serving
+- [Save and Load the compiled graph](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_sdxl_save_load.py)
+- [Change device of the compiled graph to do multi-process serving](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_sdxl_mp_load.py)
+
+### OneDiff Enterprise Edition
 If you need **Enterprise-level Support** for your system or business, you can 
 - subscribe Enterprise Edition online and get all support after the order: https://siliconflow.com/onediff.html
 - or send an email to [email protected] and tell us about your user case, deployment scale, and requirements.
@@ -70,26 +95,26 @@ OneDiff Enterprise Edition can be **subscripted for one month and one GPU** and
 
 |                      | OneDiff Enterprise   | OneDiff Community |
 | -------------------- | ------------------- | ----------- |
-| SD/SDXL series model Optimization| Yes | Yes|
-| UNet/VAE/ControlNet Optimization | Yes      | Yes         |
-| LoRA(and dynamic switching LoRA)                 | Yes             | Yes         |
-| SDXL Turbo/LCM                  | Yes             | Yes         |
-| Stable Video Diffusion |  Yes      | Yes |
-| HF diffusers            | Yes                 | Yes         |
-| ComfyUI              | Yes           | Yes         |
-| Stable Diffusion web UI | Yes          | Yes         |
 | Multiple Resolutions | Yes(No time cost for most of the cases)       | Yes(No time cost for most of the cases)           |
 | More Extreme and Dedicated optimization(usually another 20~100% performance gain)         |   Yes         |                 |
 | Technical Support for deployment    | High priority support       | Community           |
-| Get the latest technology/feature | Yes | |
+| Get the experimental technology/feature | Yes | |
+
+### Roadmap
+[OneDiff Development Roadmap](https://github.com/siliconflow/onediff/wiki#onediff-roadmap)
+
+## Community and Support
+- [Create an issue](https://github.com/siliconflow/onediff/issues)
+- Chat in Discord: [![](https://dcbadge.vercel.app/api/server/RKJTjZMcPQ?style=plastic)](https://discord.gg/RKJTjZMcPQ)
+- Email for Enterprise Edition or other business inquiries: [email protected]
 
-## OS and GPU support
+## Installation
+### OS and GPU support
 - Linux
   - If you want to use OneDiff on Windows, please use it under WSL.
 - NVIDIA GPUs
 
-## OneDiff Installation
-### Install from source
+### OneDiff Installation
 
 #### 1. Install OneFlow
 > **_NOTE:_** We have updated OneFlow a lot for OneDiff, so please install OneFlow by the links below.

diff --git a/benchmarks/pics/231228_sdxl_e2e_a100.png b/benchmarks/pics/231228_sdxl_e2e_a100.png
diff --git a/benchmarks/pics/240105_sdxl_e2e_a100.png b/benchmarks/pics/240105_sdxl_e2e_a100.png
diff --git a/imgs/0_12_sdxl.png b/imgs/0_12_sdxl.png
diff --git a/imgs/0_12_svd.png b/imgs/0_12_svd.png
diff --git a/onediff_comfy_nodes/infer_compiler_registry/register_comfy/openaimodel.py b/onediff_comfy_nodes/infer_compiler_registry/register_comfy/openaimodel.py
@@ -68,8 +68,12 @@ def forward(
         transformer_patches = transformer_options.get("patches", {})
 
         num_video_frames = kwargs.get("num_video_frames", self.default_num_video_frames)
+        default_image_only_indicator = getattr(
+            self, "default_image_only_indicator", None
+        )
+        # https://github.com/comfyanonymous/ComfyUI/blob/cb7c3a2921cfc0805be0229b4634e1143d60e6fe/comfy/ldm/modules/diffusionmodules/openaimodel.py#L832
         image_only_indicator = kwargs.get(
-            "image_only_indicator", self.default_image_only_indicator
+            "image_only_indicator", default_image_only_indicator
         )
         time_context = kwargs.get("time_context", None)
 

diff --git a/onediff_diffusers_extensions/README.md b/onediff_diffusers_extensions/README.md
@@ -3,7 +3,17 @@
 OneDiffX is a OneDiff Extension for HF diffusers. It provides some acceleration utilities, such as DeepCache.
 
 - [Install and Setup](#install-and-setup)
-- [compile_pipe](#compile_pipe)
+- [Compile, save and load pipeline](#compile-save-and-load-pipeline)
+- Acceleration for state-of-the-art Models
+  - [SDXL](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_sdxl.py)
+  - [SDXL Turbo](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_sdxl_turbo.py)
+  - [SD 1.5/2.1](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image.py)
+  - [LoRA (and dynamic switching LoRA)](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_sdxl_lora.py)
+  - [ControlNet](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_controlnet.py)
+  - [LCM](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_lcm.py) and [LCM LoRA](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_lcm_lora_sdxl.py)
+  - [Stable Video Diffusion](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/image_to_video.py)
+  - [DeepCache](https://github.com/siliconflow/onediff/blob/main/onediff_diffusers_extensions/examples/text_to_image_deep_cache_sdxl.py)
+  - [InstantID](https://github.com/siliconflow/onediff/blob/main/benchmarks/instant_id.py)
 - [DeepCache Speedup](#deepcache-speedup)
     - [Stable Diffusion XL](#run-stable-diffusion-xl-with-onediffx)
     - [Stable Diffusion 1.5](#run-stable-diffusion-15-with-onediffx)
@@ -21,9 +31,9 @@ OneDiffX is a OneDiff Extension for HF diffusers. It provides some acceleration
     git clone https://github.com/siliconflow/onediff.git
     cd onediff_diffusers_extensions && python3 -m pip install -e .
     ```
-## compile_pipe
-Compile diffusers pipeline with `compile_pipe`.
-```
+## Compile, save and load pipeline
+### Compile diffusers pipeline with `compile_pipe`.
+```python
 import torch
 from diffusers import StableDiffusionXLPipeline
 
@@ -40,6 +50,40 @@ pipe.to("cuda")
 pipe = compile_pipe(pipe)
 ```
 
+### Save compiled pipeline with `save_pipe`
+```python
+from diffusers import StableDiffusionXLPipeline
+from onediffx import compile_pipe, save_pipe
+pipe = StableDiffusionXLPipeline.from_pretrained(
+    "stabilityai/stable-diffusion-xl-base-1.0",
+    torch_dtype=torch.float16,
+    variant="fp16",
+    use_safetensors=True
+)
+pipe.to("cuda")
+
+pipe = compile_pipe(pipe)
+
+save_pipe(pipe, dir="cached_pipe")
+```
+
+### Load compiled pipeline with `load_pipe`
+```python
+from diffusers import StableDiffusionXLPipeline
+from onediffx import compile_pipe, load_pipe
+pipe = StableDiffusionXLPipeline.from_pretrained(
+    "stabilityai/stable-diffusion-xl-base-1.0",
+    torch_dtype=torch.float16,
+    variant="fp16",
+    use_safetensors=True
+)
+pipe.to("cuda")
+
+pipe = compile_pipe(pipe)
+
+load_pipe(pipe, dir="cached_pipe")
+```
+
 ## DeepCache speedup
 
 ### Run Stable Diffusion XL with OneDiffX

diff --git a/onediff_diffusers_extensions/examples/requirements.txt b/onediff_diffusers_extensions/examples/requirements.txt
@@ -1,5 +1,4 @@
 # pip install -r requirements.txt
-oneflow --pre -f https://staging.oneflow.info/branch/master/cu118
 transformers==4.27.1
 diffusers[torch]==0.19.3
 onediff

diff --git a/onediff_diffusers_extensions/onediffx/__init__.py b/onediff_diffusers_extensions/onediffx/__init__.py
@@ -1,3 +1,4 @@
+__version__ = "0.12.1.dev"
 from onediff.infer_compiler.oneflow_compiler_config import (
     oneflow_compiler_config as compiler_config,
 )

diff --git a/onediff_diffusers_extensions/onediffx/compilers/diffusion_pipeline_compiler.py b/onediff_diffusers_extensions/onediffx/compilers/diffusion_pipeline_compiler.py
@@ -67,10 +67,10 @@ def compile_pipe(
 
 
 def save_pipe(
-    pipe, dst_dir="cached_pipe", *, ignores=(), overwrite=True
+    pipe, dir="cached_pipe", *, ignores=(), overwrite=True
 ):
-    if not os.path.exists(dst_dir):
-        os.makedirs(dst_dir)
+    if not os.path.exists(dir):
+        os.makedirs(dir)
     filtered_parts = _filter_parts(ignores=ignores)
     for part in filtered_parts:
         obj = _recursive_getattr(pipe, part, None)
@@ -80,24 +80,24 @@ def save_pipe(
             and obj._deployable_module_dpl_graph is not None
             and obj.get_graph().is_compiled
         ):
-            if not overwrite and os.path.isfile(os.path.join(dst_dir, part)):
+            if not overwrite and os.path.isfile(os.path.join(dir, part)):
                 logger.info(f"Compiled graph already exists for {part}, not overwriting it.")
                 continue
             logger.info(f"Saving {part}")
-            obj.save_graph(os.path.join(dst_dir, part))
+            obj.save_graph(os.path.join(dir, part))
 
 
 def load_pipe(
-    pipe, src_dir="cached_pipe", *, ignores=(),
+    pipe, dir="cached_pipe", *, ignores=(),
 ):
-    if not os.path.exists(src_dir):
+    if not os.path.exists(dir):
         return
     filtered_parts = _filter_parts(ignores=ignores)
     for part in filtered_parts:
         obj = _recursive_getattr(pipe, part, None)
-        if obj is not None and os.path.exists(os.path.join(src_dir, part)):
+        if obj is not None and os.path.exists(os.path.join(dir, part)):
             logger.info(f"Loading {part}")
-            obj.load_graph(os.path.join(src_dir, part))
+            obj.load_graph(os.path.join(dir, part))
 
     if "image_processor" not in ignores:
         logger.info("Patching image_processor")

diff --git a/onediff_diffusers_extensions/setup.py b/onediff_diffusers_extensions/setup.py
@@ -1,8 +1,18 @@
 from setuptools import find_packages, setup
 
+
+def get_version():
+    variables = {}
+    with open("onediffx/__init__.py", "r") as f:
+        for line in f:
+            if line.startswith("__version__"):
+                exec(line, variables)
+    return variables["__version__"]
+
+
 setup(
     name="onediffx",
-    version="0.1.0",
+    version=get_version(),
     description="onediff extensions for diffusers",
     url="https://github.com/siliconflow/onediff",
     author="OneDiff contributors",
@@ -30,4 +40,6 @@
         "Programming Language :: Python :: 3.10",
         "Topic :: Scientific/Engineering :: Artificial Intelligence",
     ],
+    long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
 )
diff --git a/src/onediff/schedulers/__init__.py b/src/onediff/schedulers/__init__.py
@@ -1 +1,3 @@
-from .scheduling_euler_discrete import EulerDiscreteScheduler
+import diffusers
+
+EulerDiscreteScheduler = diffusers.EulerDiscreteScheduler