Fix default Lora/ (IA)^3 scaling in forward

setup.py

@@ -34,6 +34,7 @@
     "isort>=5.5.4",
     "Jinja2==2.11.3",
     "nltk",
+    "packaging",
     "parameterized",
     "pillow",
     "protobuf",
@@ -136,11 +137,12 @@ def deps_list(*pkgs):
 # when modifying the following list, make sure to update src/transformers/dependency_versions_check.py
 install_requires = [
     deps["transformers"],
+    deps["packaging"],
 ]
 
 setup(
     name="adapters",
-    version="1.0.1",
+    version="1.1.0.dev0",
     author="The AdapterHub team and community contributors",
     author_email="[email protected]",
     description="A Unified Library for Parameter-Efficient and Modular Transfer Learning",

src/adapters/__init__.py

@@ -16,7 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.0.1"
+__version__ = "1.1.0.dev0"
 
 from typing import TYPE_CHECKING
 

src/adapters/loading.py

@@ -6,6 +6,7 @@
 from typing import Callable, Mapping, Optional, Sequence, Tuple
 
 import torch
+from packaging.version import Version
 
 
 try:
@@ -368,6 +369,19 @@ def _rename_legacy_weights(self, k):
             k = k.replace(old, new)
         return k
 
+    def _fix_backward_compat(self, config):
+        # Fix error in previous versions for LoRA/ (IA)^3
+        if config.get("version", "").startswith("adapters.") and Version(config["version"][9:]) < Version("1.1.0"):
+            if (
+                config["config"].get("architecture", None) == "lora"
+                and config["config"]["r"] != config["config"]["alpha"]
+            ):
+                logger.warning(
+                    "Loading a LoRA trained using a faulty library version. Editing the configuration to make sure the adapter works as trained."
+                    "See https://github.com/adapter-hub/adapters/pull/770 for more."
+                )
+                config["config"]["alpha"] = config["config"]["r"]
+
     # This method is used to remove unnecessary invertible adapters from task adapters using the old format.
     # In the old format, task adapters e.g. using seq_bn config specify inv. adapters but don't use them.
     # As inv. adapters would be incorrectly used in the new implementation,
@@ -560,6 +574,8 @@ def load(
                 # The conversion to a set and then back to a list removes all duplicates
                 leave_out = list(set(leave_out + config["config"]["leave_out"]))
             config["config"]["leave_out"] = leave_out
+        # Fix issues
+        self._fix_backward_compat(config)
 
         adapter_name = load_as or config["name"]
         # If the adapter is not part of the model, add it

src/adapters/methods/lora.py

@@ -99,6 +99,7 @@ def forward(self, hidden_states: Optional[torch.Tensor], layer_input: torch.Tens
             hidden_states = hidden_states * gate
         else:
             gate = None
+            hidden_states = hidden_states * self.scaling
 
         return hidden_states, gate
 
@@ -170,6 +171,7 @@ def forward(self, hidden_states: Optional[torch.Tensor], layer_input: torch.Tens
             hidden_states = hidden_states * gate
         else:
             gate = None
+            hidden_states = hidden_states * self.scaling
 
         return hidden_states, gate