add sdxl.t2i_adapter.depth.zoe + rework mappings of t2i adapters

finegrain-ai · Oct 15, 2024 · d8f77dd · d8f77dd
1 parent 9b19272
commit d8f77dd
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 14 deletions.
diff --git a/src/refiners/conversion/cli.py b/src/refiners/conversion/cli.py
@@ -120,6 +120,7 @@ def main() -> None:
 
     # SDXL T2I-Adapters
     t2iadapter_sdxl.canny.convert()
+    t2iadapter_sdxl.depth_zoe.convert()
 
     # ELLA adapters
     ella.sd15_t5xl.convert()

diff --git a/src/refiners/conversion/models/t2iadapter_sd15.py b/src/refiners/conversion/models/t2iadapter_sd15.py
@@ -4,27 +4,28 @@
 
 diffusers_recipe = WeightRecipe(
     key_map={
+        # first part
         "adapter.conv_in": "Conv2d",
         "adapter.body.0.resnets.0.block1": "StatefulResidualBlocks_1.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_1",
-        "adapter.body.0.resnets.1.block1": "StatefulResidualBlocks_1.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
         "adapter.body.0.resnets.0.block2": "StatefulResidualBlocks_1.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_2",
+        "adapter.body.0.resnets.1.block1": "StatefulResidualBlocks_1.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
         "adapter.body.0.resnets.1.block2": "StatefulResidualBlocks_1.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_2",
-        "adapter.body.1.downsample": "StatefulResidualBlocks_2.ResidualBlocks.Downsample2d",
-        "adapter.body.2.downsample": "StatefulResidualBlocks_3.ResidualBlocks.Downsample2d",
-        "adapter.body.3.downsample": "StatefulResidualBlocks_4.ResidualBlocks.Downsample2d",
+        # second part
         "adapter.body.1.in_conv": "StatefulResidualBlocks_2.ResidualBlocks.Conv2d",
         "adapter.body.1.resnets.0.block1": "StatefulResidualBlocks_2.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_1",
-        "adapter.body.1.resnets.1.block1": "StatefulResidualBlocks_2.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
         "adapter.body.1.resnets.0.block2": "StatefulResidualBlocks_2.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_2",
+        "adapter.body.1.resnets.1.block1": "StatefulResidualBlocks_2.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
         "adapter.body.1.resnets.1.block2": "StatefulResidualBlocks_2.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_2",
+        # third part
         "adapter.body.2.in_conv": "StatefulResidualBlocks_3.ResidualBlocks.Conv2d",
         "adapter.body.2.resnets.0.block1": "StatefulResidualBlocks_3.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_1",
-        "adapter.body.2.resnets.1.block1": "StatefulResidualBlocks_3.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
-        "adapter.body.3.resnets.0.block1": "StatefulResidualBlocks_4.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_1",
-        "adapter.body.3.resnets.1.block1": "StatefulResidualBlocks_4.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
         "adapter.body.2.resnets.0.block2": "StatefulResidualBlocks_3.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_2",
+        "adapter.body.2.resnets.1.block1": "StatefulResidualBlocks_3.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
         "adapter.body.2.resnets.1.block2": "StatefulResidualBlocks_3.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_2",
+        # fourth part
+        "adapter.body.3.resnets.0.block1": "StatefulResidualBlocks_4.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_1",
         "adapter.body.3.resnets.0.block2": "StatefulResidualBlocks_4.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_2",
+        "adapter.body.3.resnets.1.block1": "StatefulResidualBlocks_4.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
         "adapter.body.3.resnets.1.block2": "StatefulResidualBlocks_4.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_2",
     },
 )

diff --git a/src/refiners/conversion/models/t2iadapter_sdxl.py b/src/refiners/conversion/models/t2iadapter_sdxl.py
@@ -4,25 +4,28 @@
 
 diffusers_recipe = WeightRecipe(
     key_map={
+        # first part
         "adapter.conv_in": "Conv2d",
         "adapter.body.0.resnets.0.block1": "StatefulResidualBlocks_1.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_1",
-        "adapter.body.0.resnets.1.block1": "StatefulResidualBlocks_1.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
         "adapter.body.0.resnets.0.block2": "StatefulResidualBlocks_1.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_2",
+        "adapter.body.0.resnets.1.block1": "StatefulResidualBlocks_1.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
         "adapter.body.0.resnets.1.block2": "StatefulResidualBlocks_1.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_2",
+        # second part
         "adapter.body.1.in_conv": "StatefulResidualBlocks_2.ResidualBlocks.Conv2d",
         "adapter.body.1.resnets.0.block1": "StatefulResidualBlocks_2.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_1",
-        "adapter.body.1.resnets.1.block1": "StatefulResidualBlocks_2.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
         "adapter.body.1.resnets.0.block2": "StatefulResidualBlocks_2.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_2",
+        "adapter.body.1.resnets.1.block1": "StatefulResidualBlocks_2.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
         "adapter.body.1.resnets.1.block2": "StatefulResidualBlocks_2.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_2",
-        "adapter.body.2.downsample": "StatefulResidualBlocks_3.ResidualBlocks.Downsample2d",
+        # third part
         "adapter.body.2.in_conv": "StatefulResidualBlocks_3.ResidualBlocks.Conv2d",
         "adapter.body.2.resnets.0.block1": "StatefulResidualBlocks_3.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_1",
-        "adapter.body.2.resnets.1.block1": "StatefulResidualBlocks_3.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
-        "adapter.body.3.resnets.0.block1": "StatefulResidualBlocks_4.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_1",
-        "adapter.body.3.resnets.1.block1": "StatefulResidualBlocks_4.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
         "adapter.body.2.resnets.0.block2": "StatefulResidualBlocks_3.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_2",
+        "adapter.body.2.resnets.1.block1": "StatefulResidualBlocks_3.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
         "adapter.body.2.resnets.1.block2": "StatefulResidualBlocks_3.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_2",
+        # fourth part
+        "adapter.body.3.resnets.0.block1": "StatefulResidualBlocks_4.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_1",
         "adapter.body.3.resnets.0.block2": "StatefulResidualBlocks_4.ResidualBlocks.Chain.ResidualBlock_1.Conv2d_2",
+        "adapter.body.3.resnets.1.block1": "StatefulResidualBlocks_4.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_1",
         "adapter.body.3.resnets.1.block2": "StatefulResidualBlocks_4.ResidualBlocks.Chain.ResidualBlock_2.Conv2d_2",
     },
 )
@@ -42,3 +45,18 @@
     recipe=diffusers_recipe,
     dtype=torch.float16,
 )
+depth_zoe = Conversion(
+    original=Hub(
+        repo_id="TencentARC/t2i-adapter-depth-zoe-sdxl-1.0",
+        filename="diffusion_pytorch_model.safetensors",
+        revision="d6282b3be9e0376d3fd0fd132959fdab143e2e51",
+        expected_sha256="613884bf550bf6750f2f22cdb17bb92f49a148cad01fdd952118df4d386acd55",
+    ),
+    converted=Hub(
+        repo_id="refiners/sdxl.t2i_adapter.depth.zoe",
+        filename="model.safetensors",
+        expected_sha256="f7199c4c6b1f38bc50ed8c7cb6220a0fd09d18779c29044e28d8ff582e941f82",
+    ),
+    recipe=diffusers_recipe,
+    dtype=torch.float16,
+)