From b6dda33f450746b4ce05ca6fe874e2e24e056ea0 Mon Sep 17 00:00:00 2001 From: calpt Date: Sun, 28 Jul 2024 18:03:31 +0200 Subject: [PATCH] Remove loading adapters from 'ah' & source parameter (#724) See https://github.com/adapter-hub/adapters/discussions/725 --- docs/classes/models/plbart.rst | 2 +- docs/huggingface_hub.md | 12 +- docs/index.rst | 1 + docs/loading.md | 36 ++-- .../configuration/adapter_fusion_config.py | 2 +- src/adapters/hub_mixin.py | 50 +---- src/adapters/loading.py | 11 +- src/adapters/model_mixin.py | 17 +- src/adapters/utils.py | 204 ++++-------------- tests/test_adapter_config.py | 9 - 10 files changed, 79 insertions(+), 265 deletions(-) diff --git a/docs/classes/models/plbart.rst b/docs/classes/models/plbart.rst index 69227f8f2e..7ce0ae54d5 100644 --- a/docs/classes/models/plbart.rst +++ b/docs/classes/models/plbart.rst @@ -1,5 +1,5 @@ PLBART -===== +====== The PLBART model was proposed in [Unified Pre-training for Program Understanding and Generation](https://arxiv.org/abs/2103.06333) by Wasi Uddin Ahmad, Saikat Chakraborty, Baishakhi Ray, Kai-Wei Chang. This is a BART-like model which can be used to perform code-summarization, code-generation, and code-translation tasks. The pre-trained model `plbart-base` has been trained using multilingual denoising task diff --git a/docs/huggingface_hub.md b/docs/huggingface_hub.md index cc1e6034ac..5c72fb194e 100644 --- a/docs/huggingface_hub.md +++ b/docs/huggingface_hub.md @@ -45,25 +45,21 @@ For more options and information, e.g. for managing models via the CLI and Git, model.push_adapter_to_hub( "my-awesome-adapter", "awesome_adapter", - adapterhub_tag="sentiment/imdb", datasets_tag="imdb" ) ``` This will create a repository `my-awesome-adapter` under your username, generate a default adapter card as `README.md` and upload the adapter named `awesome_adapter` together with the adapter card to the new repository. - `adapterhub_tag` and `datasets_tag` provide additional information for categorization. + `datasets_tag` provides additional information for categorization. ```{eval-rst} - .. important:: - All adapters uploaded to Hugging Face's Model Hub are automatically also listed on AdapterHub.ml. Thus, for better categorization, either ``adapterhub_tag`` or ``datasets_tag`` is required when uploading a new adapter to the Model Hub. - - - ``adapterhub_tag`` specifies the AdapterHub categorization of the adapter in the format ``/`` according to the tasks and subtasks shown on https://adapterhub.ml/explore. For more, see `Add a new task or subtask `_. - - ``datasets_tag`` specifies the dataset the adapter was trained on as an identifier from `Hugging Face Datasets `_. + .. note:: + All adapters uploaded to Hugging Face's Model Hub are automatically also listed on AdapterHub.ml. Thus, for better categorization, ``datasets_tag`` is helpful when uploading a new adapter to the Model Hub. ``datasets_tag`` specifies the dataset the adapter was trained on as an identifier from `Hugging Face Datasets `_. ``` Voilà! Your first adapter is on the Hugging Face Model Hub. Anyone can now run: ``` -model.load_adapter("/my-awesome-adapter", source="hf") +model.load_adapter("/my-awesome-adapter") ``` To update your adapter, simply run `push_adapter_to_hub()` with the same repository name again. This will push a new commit to the existing repository. diff --git a/docs/index.rst b/docs/index.rst index 29ef772fdf..4041e1842f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -80,6 +80,7 @@ Currently, we support the PyTorch versions of all models as listed on the `Model classes/models/gpt2 classes/models/gptj classes/models/llama + classes/models/mistral classes/models/mbart classes/models/mt5 classes/models/plbart diff --git a/docs/loading.md b/docs/loading.md index 8cd43444e0..8af81820d9 100644 --- a/docs/loading.md +++ b/docs/loading.md @@ -17,8 +17,7 @@ E.g., we can use it to retrieve information for all adapters trained for a speci ```python from adapters import list_adapters -# source can be "ah" (archived Hub repo), "hf" (huggingface.co) or None (for both, default) -adapter_infos = list_adapters(source="hf", model_name="bert-base-uncased") +adapter_infos = list_adapters(model_name="bert-base-uncased") for adapter_info in adapter_infos: print("Id:", adapter_info.adapter_id) @@ -29,7 +28,7 @@ for adapter_info in adapter_infos: In case the adapter ID is known, information for a single adapter can also be retrieved via [`get_adapter_info()`](adapters.utils.get_adapter_info): ```python -adapter_info = get_adapter_info("@ukp/bert-base-uncased_sentiment_sst-2_pfeiffer", source="ah") +adapter_info = get_adapter_info("AdapterHub/roberta-base-pf-imdb") print("Id:", adapter_info.adapter_id) print("Model name:", adapter_info.model_name) @@ -62,14 +61,14 @@ model.set_active_adapters(adapter_name) As the second example, let's have a look at how to load an adapter based on the [`AdapterInfo`](adapters.utils.AdapterInfo) returned by the [`list_adapters()`](adapters.utils.list_adapters) method from [above](#finding-pre-trained-adapters): ```python -from adapters import AutoAdapterModel, list_available_adapters +from adapters import AutoAdapterModel, list_adapters -adapter_infos = list_available_adapters(source="ah") +adapter_infos = list_adapters() # Take the first adapter info as an example adapter_info = adapter_infos[0] model = AutoAdapterModel.from_pretrained(adapter_info.model_name) -model.load_adapter(adapter_info.adapter_id, source=adapter_info.source) +model.load_adapter(adapter_info.adapter_id) ``` ### Advanced usage of `load_adapter()` @@ -78,31 +77,20 @@ To examine what's happening underneath in a bit more detail, let's first write o ```python model.load_adapter( - 'sst-2', - config='pfeiffer', - model_name='bert-base-uncased', - version=1, - load_as='sst', - source='ah' + "AdapterHub/roberta-base-pf-imdb", + version="main", + load_as="sentiment_adapter", + set_active=True, ) ``` We will go through the different arguments and their meaning one by one: -- The first argument passed to the method specifies the name of the adapter we want to load from Adapter-Hub. The library will search for an available adapter module with this name that matches the model architecture as well as the adapter type and configuration we requested. As the identifier `sst-2` resolves to a unique entry in the Hub, the corresponding adapter can be successfully loaded based on this information. To get an overview of all available adapter identifiers, please refer to [the Adapter-Hub website](https://adapterhub.ml/explore). +- The first argument passed to the method specifies the name or path from where to load the adapter. This can be the name of a repository on the [HuggingFace Model Hub](https://huggingface.co/models), a local path or a URL. To get an overview of all available adapters on the Hub, please refer to [the Adapter-Hub website](https://adapterhub.ml/explore). -- The `config` argument defines the adapter architecture the loaded adapter should have. -The value of this parameter can be either a string identifier for one of the predefined architectures, the identifier of an architecture available in the Hub or a dictionary representing a full adapter configuration. -Based on this information, the library will only search for pre-trained adapter modules having the same configuration. - -- Adapter modules trained on different pre-trained language models in general can not be used interchangeably. -Therefore, we need to make sure to load an adapter matching the language model we are using. -If possible, the library will infer the name of the pre-trained model automatically (e.g. when we use `from_pretrained('identifier')` to load a model from Hugging Face). However, if this is not the case, we must specify the name of the host model in the `model_name` parameter. - -- There could be multiple versions of the same adapter available. To load a specific version, use the `version` parameter. +- There could be multiple versions of the same adapter available as revisions in a Model Hub repository. To load a specific revision, use the `version` parameter. - By default, the `load_adapter()` method will add the loaded adapter using the identifier string given as the first argument. To load the adapter using a custom name, we can use the `load_as` parameter. -- Finally the `source` parameter provides the possibility to load adapters from alternative adapter repositories. -Besides the default value `ah`, referring to AdapterHub, it's also possible to pass `hf` to [load adapters from Hugging Face's Model Hub](huggingface_hub.md). +- Finally, `set_active` will directly activate the loaded adapter for usage in each model forward pass. Otherwise, you have to manually activate the adapter via `set_active_adapters()`. diff --git a/src/adapters/configuration/adapter_fusion_config.py b/src/adapters/configuration/adapter_fusion_config.py index 552bcdbe61..6dc31dab13 100644 --- a/src/adapters/configuration/adapter_fusion_config.py +++ b/src/adapters/configuration/adapter_fusion_config.py @@ -36,7 +36,7 @@ def load(cls, config: Union[dict, str], **kwargs): dict: The resolved adapter fusion configuration dictionary. """ # currently storing AdapterFusion weights on AdapterHub is not supported. - config_dict = resolve_adapter_config(config, local_map=ADAPTERFUSION_CONFIG_MAP, try_loading_from_hub=False) + config_dict = resolve_adapter_config(config, local_map=ADAPTERFUSION_CONFIG_MAP) # convert back to dict to allow attr overrides if isinstance(config_dict, AdapterFusionConfig): config_dict = config_dict.to_dict() diff --git a/src/adapters/hub_mixin.py b/src/adapters/hub_mixin.py index f6e0fb57f8..c23c92eb7e 100644 --- a/src/adapters/hub_mixin.py +++ b/src/adapters/hub_mixin.py @@ -1,6 +1,5 @@ import logging import os -import warnings from typing import List, Optional, Union from transformers.utils.generic import working_or_temp_dir @@ -36,7 +35,7 @@ from adapters import AutoAdapterModel model = AutoAdapterModel.from_pretrained("{model_name}") -adapter_name = model.load_adapter("{adapter_repo_name}", source="hf", set_active=True) +adapter_name = model.load_adapter("{adapter_repo_name}", set_active=True) ``` ## Architecture & Training @@ -62,7 +61,6 @@ def _save_adapter_card( save_directory: str, adapter_name: str, adapter_repo_name: str, - adapterhub_tag: Optional[str] = None, datasets_tag: Optional[str] = None, tags: Optional[List[str]] = None, language: Optional[str] = None, @@ -75,15 +73,9 @@ def _save_adapter_card( datasets = set() # Dataset/ Task info dataset_name = None - if adapterhub_tag is None and datasets_tag is None: - raise ValueError("Either adapterhub_tag or datasets_tag must be specified.") if datasets_tag is not None: dataset_name = f"[{datasets_tag}](https://huggingface.co/datasets/{datasets_tag}/)" datasets.add(datasets_tag) - if adapterhub_tag is not None: - # adapterhub_tag overwrites datasets_tag - dataset_name = f"[{adapterhub_tag}](https://adapterhub.ml/explore/{adapterhub_tag}/)" - all_tags.add(f"adapterhub:{adapterhub_tag}") all_tags.add(self.config.model_type) if tags is not None: @@ -123,10 +115,8 @@ def _save_adapter_card( def push_adapter_to_hub( self, - repo_name: str, + repo_id: str, adapter_name: str, - organization: Optional[str] = None, - adapterhub_tag: Optional[str] = None, datasets_tag: Optional[str] = None, local_path: Optional[str] = None, commit_message: Optional[str] = None, @@ -137,21 +127,15 @@ def push_adapter_to_hub( revision: str = None, commit_description: str = None, adapter_card_kwargs: Optional[dict] = None, - **deprecated_kwargs, ): """Upload an adapter to HuggingFace's Model Hub. Args: - repo_name (str): The name of the repository on the model hub to upload to. + repo_id (str): The name of the repository on the model hub to upload to. adapter_name (str): The name of the adapter to be uploaded. organization (str, optional): Organization in which to push the adapter (you must be a member of this organization). Defaults to None. - adapterhub_tag (str, optional): - Tag of the format `/` for categorization on https://adapterhub.ml/explore/. See - https://docs.adapterhub.ml/contributing.html#add-a-new-task-or-subtask for more. If not specified, - `datasets_tag` must be given in case a new adapter card is generated. Defaults to None. - datasets_tag (str, optional): Dataset identifier from https://huggingface.co/datasets. - If not specified, `adapterhub_tag` must be given in case a new adapter card is generated. Defaults to + datasets_tag (str, optional): Dataset identifier from https://huggingface.co/datasets. Defaults to None. local_path (str, optional): Local path used as clone directory of the adapter repository. If not specified, will create a temporary directory. Defaults to None. @@ -176,31 +160,6 @@ def push_adapter_to_hub( Returns: str: The url of the adapter repository on the model hub. """ - use_auth_token = deprecated_kwargs.pop("use_auth_token", None) - if use_auth_token is not None: - warnings.warn( - "The `use_auth_token` argument is deprecated and will be removed in future versions of Adapters." - " Please use `token` instead.", - FutureWarning, - ) - if token is not None: - raise ValueError( - "`token` and `use_auth_token` are both specified. Please set only the argument `token`." - ) - token = use_auth_token - - if organization is not None and not repo_name.startswith(organization): - warnings.warn( - "The `organization` argument is deprecated and will be removed in future versions of" - " Adapters. Set your organization directly in the `repo_id` passed instead" - " (`repo_id={organization}/{model_id}`)." - ) - if "/" in repo_name: - repo_name = repo_name.split("/")[-1] - repo_id = f"{organization}/{repo_name}" - else: - repo_id = repo_name - use_temp_dir = not os.path.isdir(local_path) if local_path else True # Create repo or get retrieve an existing repo @@ -218,7 +177,6 @@ def push_adapter_to_hub( work_dir, adapter_name, repo_id, - adapterhub_tag=adapterhub_tag, datasets_tag=datasets_tag, **adapter_card_kwargs, ) diff --git a/src/adapters/loading.py b/src/adapters/loading.py index d824cd24e0..b1918b0a0f 100644 --- a/src/adapters/loading.py +++ b/src/adapters/loading.py @@ -518,9 +518,9 @@ def load( - the identifier of a pre-trained task adapter to be loaded from Adapter Hub - a path to a directory containing adapter weights saved using `model.saved_adapter()` - a URL pointing to a zip folder containing a saved adapter module - config (str, optional): The requested configuration of the adapter. + config (str, optional): Deprecated. version (str, optional): The version of the adapter to be loaded. - model_name (str, optional): The string identifier of the pre-trained model. + model_name (str, optional): Deprecated. load_as (str, optional): Load the adapter using this name. By default, the name with which the adapter was saved will be used. @@ -528,6 +528,13 @@ def load( Tuple[str, str]: A tuple consisting of the local file system directory from which the weights where loaded and the name of the loaded weights. """ + # Warn about deprecated arguments + if config is not None or model_name is not None: + logger.warning( + "The 'config' and 'model_name' arguments are specific to the now unsupported legacy Hub repo and will" + " be removed." + "Please switch to only providing the HF Model Hub identifier.", + ) requested_config = AdapterConfig.load(config) if config else None # Resolve the weights to be loaded based on the given identifier and the current adapter config model_name = self.model.model_name or model_name diff --git a/src/adapters/model_mixin.py b/src/adapters/model_mixin.py index 3e174b65bd..a851eb0236 100644 --- a/src/adapters/model_mixin.py +++ b/src/adapters/model_mixin.py @@ -771,7 +771,6 @@ def load_adapter( version: str = None, model_name: str = None, load_as: str = None, - source: str = None, custom_weights_loaders: Optional[List[WeightsLoader]] = None, leave_out: Optional[List[int]] = None, id2label=None, @@ -788,20 +787,11 @@ def load_adapter( - the identifier of a pre-trained task adapter to be loaded from Adapter Hub - a path to a directory containing adapter weights saved using `model.saved_adapter()` - a URL pointing to a zip folder containing a saved adapter module - config (dict or str, optional): The requested configuration of the adapter. - If not specified, will be either: - the default adapter config for the requested adapter if specified - - the global default adapter config + config (dict or str, optional): Deprecated. version (str, optional): The version of the adapter to be loaded. - model_name (str, optional): The string identifier of the pre-trained model. + model_name (str, optional): Deprecated. load_as (str, optional): Load the adapter using this name. By default, the name with which the adapter was saved will be used. - source (str, optional): Identifier of the source(s) from where to load the adapter. Can be: - - - "ah": search on AdapterHub Hub repo. - Note: the Hub repo has been archived and all adapters have been moved to HuggingFace Model Hub. - Loading from this source is deprecated. - - "hf": search on HuggingFace Model Hub. - - None (default): search on all sources leave_out: Dynamically drop adapter modules in the specified Transformer layers when loading the adapter. set_active (bool, optional): Set the loaded adapter to be the active one. By default (False), the adapter is loaded but not @@ -818,7 +808,6 @@ def load_adapter( version, model_name, load_as, - source=source, leave_out=leave_out, set_active=set_active, **kwargs, @@ -1535,7 +1524,6 @@ def load_adapter( version: str = None, model_name: str = None, load_as: str = None, - source: str = None, with_head: bool = True, custom_weights_loaders: Optional[List[WeightsLoader]] = None, leave_out: Optional[List[int]] = None, @@ -1565,7 +1553,6 @@ def load_adapter( version=version, model_name=model_name, load_as=load_as, - source=source, custom_weights_loaders=custom_weights_loaders, leave_out=leave_out, id2label=id2label, diff --git a/src/adapters/utils.py b/src/adapters/utils.py index 25fd1da7e8..790fe7ab13 100644 --- a/src/adapters/utils.py +++ b/src/adapters/utils.py @@ -18,7 +18,6 @@ from os.path import basename, isdir, isfile, join from pathlib import Path from typing import Callable, Dict, List, Optional, Tuple, Union -from urllib.parse import urlparse from zipfile import ZipFile, is_zipfile import torch @@ -111,11 +110,11 @@ def __repr__(self): @dataclass class AdapterInfo: """ - Holds information about an adapter publicly available on AdapterHub or huggingface.co. Returned by + Holds information about an adapter publicly available on the Hub. Returned by :func:`list_adapters()`. Args: - source (str): The source repository of this adapter. Can be either "ah" (AdapterHub) or "hf" (huggingface.co). + source (str): The source repository of this adapter. Always 'hf' for adapters available on HF Model Hub. adapter_id (str): The unique identifier of this adapter. model_name (str, optional): The identifier of the model this adapter was trained for. task (str, optional): The task this adapter was trained for. @@ -434,7 +433,7 @@ def parse_adapter_config_string(config_string: str) -> List[Tuple[str, dict]]: return adapter_configs -def resolve_adapter_config(config: Union[dict, str], local_map=None, try_loading_from_hub=True, **kwargs) -> dict: +def resolve_adapter_config(config: Union[dict, str], local_map=None, **kwargs) -> dict: """ Resolves a given adapter configuration specifier to a full configuration dictionary. @@ -444,7 +443,6 @@ def resolve_adapter_config(config: Union[dict, str], local_map=None, try_loading - a dictionary: returned without further action - an identifier string available in local_map - the path to a file containing a full adapter configuration - - an identifier string available in Adapter-Hub Returns: dict: The resolved adapter configuration dictionary. @@ -464,13 +462,6 @@ def resolve_adapter_config(config: Union[dict, str], local_map=None, try_loading return loaded_config["config"] else: return loaded_config - # download hub index file - if try_loading_from_hub: - index_file = download_cached(ADAPTER_HUB_CONFIG_FILE, **kwargs) - if not index_file: - raise EnvironmentError("Unable to load adapter hub index file. The file might be temporarily unavailable.") - with open(index_file, "r") as f: - config_index = json.load(f) # parse the config string config_pairs = parse_adapter_config_string(config) if len(config_pairs) > 0: @@ -480,11 +471,6 @@ def resolve_adapter_config(config: Union[dict, str], local_map=None, try_loading if local_map and name in local_map: config_obj = local_map[name] full_configs.append(config_obj.replace(**config_kwargs)) - # now, try to find in hub index - elif try_loading_from_hub and name in config_index: - config_obj = config_index[name] - config_obj.update(**config_kwargs) - full_configs.append(config_obj) else: raise ValueError("Could not identify '{}' as a valid adapter configuration.".format(name)) # Case 1: only one config, return it directly @@ -588,34 +574,16 @@ def _get_matching_version(config_entry, org): raise ValueError("Multiple adapters with this name are available for this config.") -def http_get_json(url): - # check if it's a relative url - if not urlparse(url).netloc: - url = urljoin(ADAPTER_HUB_URL, url) - response = requests.get(url) - if response.status_code == 200: - return response.json() - else: - raise EnvironmentError("Failed to get file {}".format(url)) - - -def get_checksum(file_entry: dict): - for algo in hashlib.algorithms_guaranteed: - if algo in file_entry: - return algo, file_entry[algo] - - def pull_from_hub( specifier: str, model_name: str, adapter_config: Optional[Union[dict, str]] = None, version: str = None, strict: bool = False, - redirect_to_hf_hub: bool = False, **kwargs, ) -> str: """ - Downloads a pre-trained adapter module from Adapter-Hub + Redirects loading from the archived Hub repository to HuggingFace Model Hub. Args: specifier (str): A string specifying the adapter to be loaded. @@ -624,9 +592,6 @@ def pull_from_hub( version (str, optional): The version of the adapter to be loaded. Defaults to None. strict (bool, optional): If set to True, only allow adapters exactly matching the given config to be loaded. Defaults to False. - redirect_to_hf_hub (bool, optional): - If set to True, the function will redirect to the HuggingFace Model Hub instead of AdapterHub. - Defaults to False. Returns: str: The local path to which the adapter has been downloaded. @@ -642,35 +607,12 @@ def pull_from_hub( raise EnvironmentError("No adapter with name '{}' was found in the adapter index.".format(specifier)) hf_hub_specifier = "AdapterHub/" + os.path.basename(hub_entry_url).split(".")[0] - if redirect_to_hf_hub: - logger.warning( - "Automatic redirect to HF Model Hub repo '{}'. Please switch to the new ID to remove this warning.".format( - hf_hub_specifier - ) - ) - return pull_from_hf_model_hub(hf_hub_specifier, version=version, **kwargs) - else: - logger.warning( - "Loading adapters from this source is deprecated. This adapter has moved to '{}'. Please switch to the new" - " ID to remove this warning.".format(hf_hub_specifier) + logger.warning( + "Automatic redirect to HF Model Hub repo '{}'. Please switch to the new ID to remove this warning.".format( + hf_hub_specifier ) - - hub_entry = http_get_json(hub_entry_url) - # set version - if not version: - version = hub_entry["default_version"] - elif version not in hub_entry["files"]: - logger.warning("Version '{}' of adapter '{}' not found. Falling back to default.".format(version, specifier)) - version = hub_entry["default_version"] - file_entry = hub_entry["files"][version] - - # start downloading - logger.info("Resolved adapter files at {}.".format(file_entry["url"])) - checksum_algo, checksum = get_checksum(file_entry) - download_path = download_cached(file_entry["url"], checksum=checksum, checksum_algo=checksum_algo, **kwargs) - if not download_path: - raise EnvironmentError("Unable to load file from {}. The file might be unavailable.".format(file_entry["url"])) - return download_path + ) + return pull_from_hf_model_hub(hf_hub_specifier, version=version, **kwargs) def pull_from_hf_model_hub(specifier: str, version: str = None, **kwargs) -> str: @@ -689,8 +631,6 @@ def resolve_adapter_path( model_name: str = None, adapter_config: Union[dict, str] = None, version: str = None, - source: str = None, - redirect_to_hf_hub: bool = False, **kwargs, ) -> str: """ @@ -706,15 +646,6 @@ def resolve_adapter_path( model_name (str, optional): The identifier of the pre-trained model for which to load an adapter. adapter_config (Union[dict, str], optional): The configuration of the adapter to be loaded. version (str, optional): The version of the adapter to be loaded. Defaults to None. - source (str, optional): Identifier of the source(s) from where to get adapters. Can be either: - - - "ah": search on AdapterHub.ml. Note: this source is deprecated in favor of "hf". - - "hf": search on HuggingFace model hub (huggingface.co). - - None (default): search on all sources - - redirect_to_hf_hub (bool, optional): - If set to True, the function will redirect to the HuggingFace Model Hub instead of AdapterHub. - Defaults to False. Returns: str: The local path from where the adapter module can be loaded. @@ -739,24 +670,13 @@ def resolve_adapter_path( WEIGHTS_NAME, CONFIG_NAME, adapter_name_or_path ) ) - elif source == "ah": - return pull_from_hub( - adapter_name_or_path, - model_name, - adapter_config=adapter_config, - version=version, - redirect_to_hf_hub=redirect_to_hf_hub, - **kwargs, - ) - elif source == "hf": - return pull_from_hf_model_hub(adapter_name_or_path, version=version, **kwargs) - elif source is None: + else: try: - logger.info("Attempting to load adapter from source 'hf'...") + logger.info("Attempting to load adapter from HF Model Hub...") return pull_from_hf_model_hub(adapter_name_or_path, version=version, **kwargs) except (EnvironmentError, ValueError) as ex: logger.info(ex) - logger.info("Attempting to load adapter from source 'ah'...") + logger.info("Attempting to redirect from archived Hub repo...") try: return pull_from_hub( adapter_name_or_path, @@ -773,97 +693,63 @@ def resolve_adapter_path( adapter_name_or_path ) ) - else: - raise ValueError("Unable to identify {} as a valid module location.".format(adapter_name_or_path)) -def list_adapters(source: str = None, model_name: str = None) -> List[AdapterInfo]: +def list_adapters(model_name: str = None) -> List[AdapterInfo]: """ Retrieves a list of all publicly available adapters on AdapterHub.ml or on huggingface.co. Args: - source (str, optional): Identifier of the source(s) from where to get adapters. Can be either: - - - "ah": search on AdapterHub.ml. - - "hf": search on HuggingFace model hub (huggingface.co). - - None (default): search on all sources - model_name (str, optional): If specified, only returns adapters trained for the model with this identifier. """ adapters = [] - if source == "ah" or source is None: - try: - all_ah_adapters_file = download_cached(ADAPTER_HUB_ALL_FILE) - except requests.exceptions.HTTPError: - raise EnvironmentError( - "Unable to load list of adapters from AdapterHub.ml. The service might be temporarily unavailable." - ) - with open(all_ah_adapters_file, "r") as f: - all_ah_adapters_data = json.load(f) - adapters += [AdapterInfo(**info) for info in all_ah_adapters_data] - if source == "hf" or source is None: - if "fetch_config" in inspect.signature(HfApi.list_models).parameters: - kwargs = {"full": True, "fetch_config": True} - else: - logger.warning( - "Using old version of huggingface-hub package for fetching. Please upgrade to latest version for" - " accurate results." - ) - kwargs = {"full": True} - all_hf_adapters_data = HfApi().list_models(filter="adapters", **kwargs) - for model_info in all_hf_adapters_data: - adapter_info = AdapterInfo( - source="hf", - adapter_id=model_info.modelId, - model_name=model_info.config.get("adapters", {}).get("model_name") if model_info.config else None, - username=model_info.modelId.split("/")[0], - sha1_checksum=model_info.sha, - ) - adapters.append(adapter_info) + if "fetch_config" in inspect.signature(HfApi.list_models).parameters: + kwargs = {"full": True, "fetch_config": True} + else: + logger.warning( + "Using old version of huggingface-hub package for fetching. Please upgrade to latest version for" + " accurate results." + ) + kwargs = {"full": True} + all_hf_adapters_data = HfApi().list_models(filter="adapters", **kwargs) + for model_info in all_hf_adapters_data: + adapter_info = AdapterInfo( + source="hf", + adapter_id=model_info.modelId, + model_name=model_info.config.get("adapters", {}).get("model_name") if model_info.config else None, + username=model_info.modelId.split("/")[0], + sha1_checksum=model_info.sha, + ) + adapters.append(adapter_info) if model_name is not None: adapters = [adapter for adapter in adapters if adapter.model_name == model_name] return adapters -def get_adapter_info(adapter_id: str, source: str = "ah") -> Optional[AdapterInfo]: +def get_adapter_info(adapter_id: str) -> Optional[AdapterInfo]: """ Retrieves information about a specific adapter. Args: adapter_id (str): The identifier of the adapter to retrieve. - source (str, optional): Identifier of the source(s) from where to get adapters. Can be either: - - - "ah": search on AdapterHub.ml. - - "hf": search on HuggingFace model hub (huggingface.co). Returns: AdapterInfo: The adapter information or None if the adapter was not found. """ - if source == "ah": - if adapter_id.startswith("@"): - adapter_id = adapter_id[1:] - try: - data = http_get_json(f"/adapters/{adapter_id}.json") - return AdapterInfo(**data["info"]) - except EnvironmentError: - return None - elif source == "hf": - try: - model_info = HfApi().model_info(adapter_id) - return AdapterInfo( - source="hf", - adapter_id=model_info.modelId, - model_name=( - model_info.config.get("adapter_transformers", {}).get("model_name") if model_info.config else None - ), - username=model_info.modelId.split("/")[0], - sha1_checksum=model_info.sha, - ) - except requests.exceptions.HTTPError: - return None - else: - raise ValueError("Please specify either 'ah' or 'hf' as source.") + try: + model_info = HfApi().model_info(adapter_id) + return AdapterInfo( + source="hf", + adapter_id=model_info.modelId, + model_name=( + model_info.config.get("adapter_transformers", {}).get("model_name") if model_info.config else None + ), + username=model_info.modelId.split("/")[0], + sha1_checksum=model_info.sha, + ) + except requests.exceptions.HTTPError: + return None def prefix_attention_mask(attention_mask, dim: Union[int, List[int]] = 3, prefix_value: int = 0): diff --git a/tests/test_adapter_config.py b/tests/test_adapter_config.py index 2bce31c7e3..db57aeae2b 100644 --- a/tests/test_adapter_config.py +++ b/tests/test_adapter_config.py @@ -18,15 +18,6 @@ @require_torch class AdapterConfigTest(unittest.TestCase): - def test_config_load(self): - download_kwargs = {"force_download": True} - # TODO still uses the old config names as only these are available on the Hub - for config_name in ["pfeiffer", "houlsby"]: - with self.subTest(config_name=config_name): - config = AdapterConfig.load(config_name, download_kwargs=download_kwargs, non_linearity="leakyrelu") - self.assertTrue(isinstance(config, AdapterConfig)) - self.assertEqual(config.non_linearity, "leakyrelu") - def test_config_immutable(self): def set_attr(config: AdapterConfig): config.non_linearity = "dummy"