From 80f9d6682488f9917127730718b898bf1ec8c177 Mon Sep 17 00:00:00 2001
From: claudiofr <claudiofr@yahoo.com>
Date: Sun, 15 Jan 2023 11:27:39 -0500
Subject: [PATCH 01/15] Change muled resolution beaker cache type from file to
 ext:database.

Rather than store cache data in the file system store cache data
in the database. Partially addresses issue 15216. For now this was
only changed for the mulled resolution cache. Introduced new
config parameters for database url, cache table name. Introduced
logic to default values of a parameter to values of another
parameter. i.e. default cache database url to database_connection.
Metadata for this was placed in code. Should probably be
placed eventually in config file which would require a change
to the config_schema.yml.

Introduced a new unit test, test_mulled_resolution_cache_db.py.
---
 doc/source/admin/galaxy_options.rst           | 37 +++++++++++++
 lib/galaxy/app.py                             |  3 ++
 lib/galaxy/config/__init__.py                 | 23 ++++++++
 lib/galaxy/config/sample/galaxy.yml.sample    | 18 ++++++-
 lib/galaxy/config/schemas/config_schema.yml   | 21 +++++++-
 .../tool_util/test_resolution_cache_db.py     | 54 +++++++++++++++++++
 6 files changed, 154 insertions(+), 2 deletions(-)
 create mode 100644 test/unit/tool_util/test_resolution_cache_db.py

diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst
index 63a83293b9d6..be87a633fd4f 100644
--- a/doc/source/admin/galaxy_options.rst
+++ b/doc/source/admin/galaxy_options.rst
@@ -1347,6 +1347,43 @@
 :Type: int
 
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``mulled_resolution_cache_url``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    When mulled_resolution_cache_type = ext:database, this is
+    the url of the database used by beaker for caching mulled resolution
+    requests. The application config code will set it to the 
+    value of database_connection if this is not set.
+:Default: ``None``
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``mulled_resolution_cache_schema_name``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    When mulled_resolution_cache_type = ext:database, this is
+    the database schema name of the table used by beaker for
+    caching mulled resolution requests. 
+:Default: ``None``
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``mulled_resolution_cache_table_name``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    When mulled_resolution_cache_type = ext:database, this is
+    the database table name used by beaker for
+    caching mulled resolution requests. 
+:Default: ``mulled_resolution_beaker_cache``
+:Type: str
+
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ``object_store_config_file``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py
index 3b1796ac0951..767cc27bbd07 100644
--- a/lib/galaxy/app.py
+++ b/lib/galaxy/app.py
@@ -323,6 +323,9 @@ def _configure_toolbox(self):
                 "cache.data_dir": self.config.mulled_resolution_cache_data_dir,
                 "cache.lock_dir": self.config.mulled_resolution_cache_lock_dir,
                 "cache.expire": self.config.mulled_resolution_cache_expire,
+                "cache.url": self.config.mulled_resolution_cache_url,
+                "cache.table_name": self.config.mulled_resolution_cache_table_name,
+                "cache.schema_name": self.config.mulled_resolution_cache_schema_name,
             }
             mulled_resolution_cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache(
                 "mulled_resolution"
diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py
index a70fb9e3acf2..01f839506668 100644
--- a/lib/galaxy/config/__init__.py
+++ b/lib/galaxy/config/__init__.py
@@ -139,6 +139,13 @@
 }
 """Default value for logging configuration, passed to :func:`logging.config.dictConfig`"""
 
+DEPENDENT_CONFIG_DEFAULTS: Dict[str, str] = {
+    "mulled_resolution_cache_url": "database_connection",
+}
+"""Config parameters whose default is the value of another config parameter
+This should be moved to a .yml config file.
+"""
+
 VERSION_JSON_FILE = "version.json"
 DEFAULT_EMAIL_FROM_LOCAL_PART = "galaxy-no-reply"
 DISABLED_FLAG = "disabled"  # Used to mark a config option as disabled
@@ -731,6 +738,22 @@ def __init__(self, **kwargs):
         self._override_tempdir(kwargs)
         self._configure_sqlalchemy20_warnings(kwargs)
         self._process_config(kwargs)
+        self._set_dependent_defaults()
+
+    def _set_dependent_defaults(self):
+        """Set values of unset parameters which take their default values from other parameters"""
+        for dependent_config_param, config_param in DEPENDENT_CONFIG_DEFAULTS.items():
+            try:
+                if getattr(self, dependent_config_param) is None:
+                    db_connect = getattr(self, config_param)
+                    setattr(self, dependent_config_param, getattr(self, config_param))
+            except AttributeError:
+                log.warning(
+                    "One or more invalid config parameter names specified in "
+                    "DEPENDENT_CONFIG_DEFAULTS, "
+                    f"{dependent_config_param}, {config_param}"
+                )
+
 
     def _configure_sqlalchemy20_warnings(self, kwargs):
         """
diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample
index 927736e0f4eb..c88e1a63ca73 100644
--- a/lib/galaxy/config/sample/galaxy.yml.sample
+++ b/lib/galaxy/config/sample/galaxy.yml.sample
@@ -963,7 +963,7 @@ galaxy:
   # Mulled resolution caching. Mulled resolution uses external APIs of
   # quay.io, these requests are caching using this and the following
   # parameters
-  #mulled_resolution_cache_type: file
+  #mulled_resolution_cache_type: ext:database
 
   # Data directory used by beaker for caching mulled resolution
   # requests.
@@ -981,6 +981,22 @@ galaxy:
   # created.
   #mulled_resolution_cache_expire: 3600
 
+  # When mulled_resolution_cache_type = ext:database, this is
+  # the url of the database used by beaker for caching mulled resolution
+  # requests. The application config code will set it to the 
+  # value of database_connection if this is not set.
+  #mulled_resolution_cache_url: null
+
+  # When mulled_resolution_cache_type = ext:database, this is
+  # the database schema name of the table used by beaker for
+  # caching mulled resolution requests. 
+  #mulled_resolution_cache_schema_name: null
+
+  # When mulled_resolution_cache_type = ext:database, this is
+  # the database table name used by beaker for
+  # caching mulled resolution requests. 
+  #mulled_resolution_cache_table_name: null
+
   # Configuration file for the object store If this is set and exists,
   # it overrides any other objectstore settings.
   # The value of this option will be resolved with respect to
diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml
index e8b75067f90c..9e4c3bf6dcc6 100644
--- a/lib/galaxy/config/schemas/config_schema.yml
+++ b/lib/galaxy/config/schemas/config_schema.yml
@@ -936,7 +936,7 @@ mapping:
 
       mulled_resolution_cache_type:
         type: str
-        default: file
+        default: ext:database
         required: false
         desc: |
           Mulled resolution caching. Mulled resolution uses external APIs of quay.io, these
@@ -965,6 +965,25 @@ mapping:
         desc: |
           Seconds until the beaker cache is considered old and a new value is created.
 
+      mulled_resolution_cache_url:
+        type: str
+        required: false
+        desc: |
+          Url to database used by beaker for caching mulled resolution requests.
+
+      mulled_resolution_cache_table_name:
+        type: str
+        default: mulled_resolution_beaker_cache
+        required: false
+        desc: |
+          Database table used by beaker for caching mulled resolution requests.
+
+      mulled_resolution_cache_schema_name:
+        type: str
+        required: false
+        desc: |
+          Schema of database table used by beaker for caching mulled resolution requests.
+
       object_store_config_file:
         type: str
         default: object_store_conf.xml
diff --git a/test/unit/tool_util/test_resolution_cache_db.py b/test/unit/tool_util/test_resolution_cache_db.py
new file mode 100644
index 000000000000..26d2d08a360f
--- /dev/null
+++ b/test/unit/tool_util/test_resolution_cache_db.py
@@ -0,0 +1,54 @@
+import time
+
+import pytest
+from beaker.cache import CacheManager
+from beaker.util import parse_cache_config_options
+
+from galaxy.tool_util.deps.container_resolvers import ResolutionCache
+from galaxy.tool_util.deps.mulled.util import (
+    _namespace_has_repo_name,
+    mulled_tags_for,
+    NAMESPACE_HAS_REPO_NAME_KEY,
+    TAG_CACHE_KEY,
+)
+
+from galaxy import config
+
+@pytest.fixture()
+def appconfig():
+    return config.GalaxyAppConfiguration(override_tempdir=False)
+
+@pytest.fixture()
+def resolution_cache(tmpdir, appconfig):
+    resolution_cache = ResolutionCache()
+    cache_opts = {
+        "cache.type": "ext:database",
+        "cache.lock_dir": str(tmpdir / "lock"),
+        "cache.expire": "1",
+        "cache.url": appconfig.mulled_resolution_cache_url,
+        "cache.schema_name" : appconfig.mulled_resolution_cache_schema_name,
+        "cache.table_name" : appconfig.mulled_resolution_cache_table_name,
+    }
+    cm = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("mulled_resolution")
+    resolution_cache.mulled_resolution_cache = cm
+    return resolution_cache
+
+
+def test_resolution_cache_namepace_has_repo_name(resolution_cache):
+    resolution_cache.mulled_resolution_cache[NAMESPACE_HAS_REPO_NAME_KEY] = ["mytool3000"]
+    assert _namespace_has_repo_name("bioconda", "mytool3000", resolution_cache=resolution_cache)
+
+
+def test_resolution_cache_expires(resolution_cache):
+    resolution_cache.mulled_resolution_cache[NAMESPACE_HAS_REPO_NAME_KEY] = ["mytool3000"]
+    assert NAMESPACE_HAS_REPO_NAME_KEY in resolution_cache.mulled_resolution_cache
+    time.sleep(1.2)
+    assert NAMESPACE_HAS_REPO_NAME_KEY not in resolution_cache.mulled_resolution_cache
+
+
+def test_targets_to_mulled_name(resolution_cache):
+    resolution_cache.mulled_resolution_cache[NAMESPACE_HAS_REPO_NAME_KEY] = ["mytool3000"]
+    cache = resolution_cache.mulled_resolution_cache._get_cache("mulled_tag_cache", {"expire": 1})
+    cache[TAG_CACHE_KEY] = {"bioconda": {"mytool3000": ["1.0", "1.1"]}}
+    tags = mulled_tags_for(namespace="bioconda", image="mytool3000", resolution_cache=resolution_cache)
+    assert tags == ["1.1", "1.0"]

From 213398aee35466cbe3e7a6b553fcbcc277076740 Mon Sep 17 00:00:00 2001
From: claudiofr <claudiofr@yahoo.com>
Date: Sat, 21 Jan 2023 13:39:14 -0500
Subject: [PATCH 02/15] Change citation and biotools_service beaker cache type
 from file to ext:database.

Rather than store cache data in the file system store cache data
in the database. These are the last 2 beaker caches that need
to be converted to the database as part of issue 15216.
Introduce new config parameters for database url, cache table name.
Introduce logic to default values of a parameter to values of another
parameter. i.e. default cache database url to database_connection.
Metadata for this was placed in code. Should probably be
placed eventually in config file which would require a change
to the config_schema.yml.

Introduced a new unit test, test_citations_db.py
Did not introduce a new test for the biotools_service beaker cache
because I could not find a place in the code where it was used.
---
 doc/source/admin/galaxy_options.rst           | 81 ++++++++++++++++++-
 lib/galaxy/config/__init__.py                 |  2 +
 lib/galaxy/config/sample/galaxy.yml.sample    | 38 ++++++++-
 lib/galaxy/config/schemas/config_schema.yml   | 45 ++++++++++-
 lib/galaxy/managers/citations.py              |  5 +-
 lib/galaxy/tools/biotools.py                  |  5 +-
 .../app/managers/test_CitationsManager.py     |  1 +
 .../app/managers/test_CitationsManager_db.py  | 15 ++++
 8 files changed, 183 insertions(+), 9 deletions(-)
 create mode 100644 test/unit/app/managers/test_CitationsManager_db.py

diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst
index be87a633fd4f..4e92fa0151a2 100644
--- a/doc/source/admin/galaxy_options.rst
+++ b/doc/source/admin/galaxy_options.rst
@@ -1225,7 +1225,7 @@
 :Description:
     bio.tools web service request related caching. The type of beaker
     cache used.
-:Default: ``file``
+:Default: ``ext:database``
 :Type: str
 
 
@@ -1255,6 +1255,44 @@
 :Type: str
 
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``biotools_service_cache_url``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    When biotools_service_cache_type = ext:database, this is
+    the url of the database used by beaker for
+    bio.tools web service request related caching.
+    The application config code will set it to the 
+    value of database_connection if this is not set.
+:Default: ``None``
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``biotools_service_cache_schema_name``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    When biotools_service_cache_type = ext:database, this is
+    the database schema name of the table used by beaker for
+    bio.tools web service request related caching.
+:Default: ``None``
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``biotools_service_cache_table_name``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    When biotools_service_cache_type = ext:database, this is
+    the database table name used by beaker for
+    bio.tools web service request related caching.
+:Default: ``biotools_service_beaker_cache``
+:Type: str
+
+
 ~~~~~~~~~~~~~~~~~~~~~~~
 ``citation_cache_type``
 ~~~~~~~~~~~~~~~~~~~~~~~
@@ -1264,7 +1302,7 @@
     fetched from external sources such as https://doi.org/ by Galaxy -
     the following parameters can be used to control the caching used
     to store this information.
-:Default: ``file``
+:Default: ``ext:database``
 :Type: str
 
 
@@ -1298,6 +1336,43 @@
 :Type: str
 
 
+~~~~~~~~~~~~~~~~~~~~~~
+``citation_cache_url``
+~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    When citation_cache_type = ext:database, this is
+    the url of the database used by beaker for citation
+    caching. The application config code will set it to the 
+    value of database_connection if this is not set.
+:Default: ``None``
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``citation_cache_schema_name``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    When citation_cache_type = ext:database, this is
+    the database schema name of the table used by beaker for
+    citation related caching.
+:Default: ``None``
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``citation_cache_table_name``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    When citation_cache_type = ext:database, this is
+    the database table name used by beaker for
+    citation related caching.
+:Default: ``citation_beaker_cache``
+:Type: str
+
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ``mulled_resolution_cache_type``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1306,7 +1381,7 @@
     Mulled resolution caching. Mulled resolution uses external APIs of
     quay.io, these requests are caching using this and the following
     parameters
-:Default: ``file``
+:Default: ``ext:database``
 :Type: str
 
 
diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py
index 01f839506668..0ca4c5ed5eff 100644
--- a/lib/galaxy/config/__init__.py
+++ b/lib/galaxy/config/__init__.py
@@ -141,6 +141,8 @@
 
 DEPENDENT_CONFIG_DEFAULTS: Dict[str, str] = {
     "mulled_resolution_cache_url": "database_connection",
+    "citation_cache_url": "database_connection",
+    "biotools_service_cache_url": "database_connection",
 }
 """Config parameters whose default is the value of another config parameter
 This should be moved to a .yml config file.
diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample
index c88e1a63ca73..a709bdbfb847 100644
--- a/lib/galaxy/config/sample/galaxy.yml.sample
+++ b/lib/galaxy/config/sample/galaxy.yml.sample
@@ -924,7 +924,7 @@ galaxy:
 
   # bio.tools web service request related caching. The type of beaker
   # cache used.
-  #biotools_service_cache_type: file
+  #biotools_service_cache_type: ext:database
 
   # bio.tools web service request related caching. The data directory to
   # point beaker cache at.
@@ -938,11 +938,28 @@ galaxy:
   # <cache_dir>.
   #biotools_service_cache_lock_dir: biotools/locks
 
+  # When biotools_service_cache_type = ext:database, this is
+  # the url of the database used by beaker
+  # for bio.tools web service request caching.
+  # The application config code will set it to the 
+  # value of database_connection if this is not set.
+  #biotools_service_cache_url: null
+
+  # When biotools_service_cache_type = ext:database, this is
+  # the database schema name of the table used by beaker
+  # for bio.tools web service request caching.
+  #biotools_service_cache_schema_name: null
+
+  # When biotools_service_cache_type = ext:database, this is
+  # the database table name used by beaker
+  # for bio.tools web service request caching.
+  #biotools_service_cache_table_name: null
+
   # Citation related caching.  Tool citations information maybe fetched
   # from external sources such as https://doi.org/ by Galaxy - the
   # following parameters can be used to control the caching used to
   # store this information.
-  #citation_cache_type: file
+  #citation_cache_type: ext:database
 
   # Citation related caching.  Tool citations information maybe fetched
   # from external sources such as https://doi.org/ by Galaxy - the
@@ -960,6 +977,23 @@ galaxy:
   # <cache_dir>.
   #citation_cache_lock_dir: citations/locks
 
+  # When citation_cache_type = ext:database, this is
+  # the url of the database used by beaker for citation
+  # related caching.
+  # The application config code will set it to the 
+  # value of database_connection if this is not set.
+  #citation_cache_url: null
+
+  # When citation_cache_type = ext:database, this is
+  # the database schema name of the table used by beaker
+  # for citation related caching.
+  #citation_cache_schema_name: null
+
+  # When citation_cache_type = ext:database, this is
+  # the database table name used by beaker
+  # for citation related caching.
+  #citation_cache_table_name: null
+
   # Mulled resolution caching. Mulled resolution uses external APIs of
   # quay.io, these requests are caching using this and the following
   # parameters
diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml
index 9e4c3bf6dcc6..ef153d1ac8a5 100644
--- a/lib/galaxy/config/schemas/config_schema.yml
+++ b/lib/galaxy/config/schemas/config_schema.yml
@@ -882,7 +882,7 @@ mapping:
 
       biotools_service_cache_type:
         type: str
-        default: file
+        default: ext:database
         required: false
         desc: |
           bio.tools web service request related caching. The type of beaker cache used.
@@ -905,9 +905,31 @@ mapping:
           bio.tools web service request related caching. The lock directory to point
           beaker cache at.
 
+      biotools_service_cache_url:
+        type: str
+        required: false
+        desc: |
+          Url to database used by beaker for bio.tools web service request 
+          related caching.
+
+      biotools_service_cache_table_name:
+        type: str
+        default: biotools_service_beaker_cache
+        required: false
+        desc: |
+          Database table used by beaker for bio.tools web service request 
+          related caching.
+
+      biotools_service_cache_schema_name:
+        type: str
+        required: false
+        desc: |
+          Schema of database table used by beaker for bio.tools web service request 
+          related caching.
+
       citation_cache_type:
         type: str
-        default: file
+        default: ext:database
         required: false
         desc: |
           Citation related caching.  Tool citations information maybe fetched from
@@ -934,6 +956,25 @@ mapping:
           external sources such as https://doi.org/ by Galaxy - the following
           parameters can be used to control the caching used to store this information.
 
+      citation_cache_url:
+        type: str
+        required: false
+        desc: |
+          Url to database used by beaker for citation related caching.
+
+      citation_cache_table_name:
+        type: str
+        default: citation_beaker_cache
+        required: false
+        desc: |
+          Database table used by beaker for citation related caching.
+
+      citation_cache_schema_name:
+        type: str
+        required: false
+        desc: |
+          Schema of database table used by beaker for citation related caching.
+
       mulled_resolution_cache_type:
         type: str
         default: ext:database
diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py
index d8f2264b1b1e..451994fe93b3 100644
--- a/lib/galaxy/managers/citations.py
+++ b/lib/galaxy/managers/citations.py
@@ -38,9 +38,12 @@ def _get_tool(self, tool_id):
 class DoiCache:
     def __init__(self, config):
         cache_opts = {
-            "cache.type": getattr(config, "citation_cache_type", "file"),
+            "cache.type": getattr(config, "citation_cache_type", "ext:database"),
             "cache.data_dir": getattr(config, "citation_cache_data_dir", None),
             "cache.lock_dir": getattr(config, "citation_cache_lock_dir", None),
+            "cache.url": getattr(config, "citation_cache_url", None),
+            "cache.table_name": getattr(config, "citation_cache_table_name", "citation_beaker_cache"),
+            "cache.schema_name": getattr(config, "citation_cache_schema_name", None),
         }
         self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi")
 
diff --git a/lib/galaxy/tools/biotools.py b/lib/galaxy/tools/biotools.py
index d396037b1381..e8ce4d303bb7 100644
--- a/lib/galaxy/tools/biotools.py
+++ b/lib/galaxy/tools/biotools.py
@@ -16,9 +16,12 @@ def get_galaxy_biotools_metadata_source(config) -> BiotoolsMetadataSource:
     biotools_metadata_source_config.content_directory = config.biotools_content_directory
     biotools_metadata_source_config.use_api = config.biotools_use_api
     cache_opts = {
-        "cache.type": getattr(config, "biotools_service_cache_type", "file"),
+        "cache.type": getattr(config, "biotools_service_cache_type", "ext:database"),
         "cache.data_dir": getattr(config, "biotools_service_cache_data_dir", None),
         "cache.lock_dir": getattr(config, "biotools_service_cache_lock_dir", None),
+        "cache.url": getattr(config, "biotools_service_cache_url", config.database_connection),
+        "cache.table_name": getattr(config, "biotools_service_cache_table_name", "biotools_service_beaker_cache"),
+        "cache.schema_name": getattr(config, "biotools_service_cache_schema_name", None),
     }
     cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi")
     biotools_metadata_source_config.cache = cache
diff --git a/test/unit/app/managers/test_CitationsManager.py b/test/unit/app/managers/test_CitationsManager.py
index 9a739c86d86b..030fc6de489c 100644
--- a/test/unit/app/managers/test_CitationsManager.py
+++ b/test/unit/app/managers/test_CitationsManager.py
@@ -8,6 +8,7 @@
 def test_DoiCache():
     with tempfile.TemporaryDirectory() as tmp_database_dir:
         config = Bunch(
+            citation_cache_type="file",
             citation_cache_data_dir=os.path.join(tmp_database_dir, "data"),
             citation_cache_lock_dir=os.path.join(tmp_database_dir, "locks"),
         )
diff --git a/test/unit/app/managers/test_CitationsManager_db.py b/test/unit/app/managers/test_CitationsManager_db.py
new file mode 100644
index 000000000000..e2a8758d033c
--- /dev/null
+++ b/test/unit/app/managers/test_CitationsManager_db.py
@@ -0,0 +1,15 @@
+import os.path
+import tempfile
+import pytest
+
+from galaxy.managers.citations import DoiCache
+from galaxy.util.bunch import Bunch
+from galaxy import config
+
+@pytest.fixture()
+def doi_cache():
+    return DoiCache(config.GalaxyAppConfiguration(override_tempdir=False))
+
+def test_DoiCache(doi_cache):
+    assert "Jörg" in doi_cache.get_bibtex("10.1093/bioinformatics/bts252")
+    assert "Özkurt" in doi_cache.get_bibtex("10.1101/2021.12.24.474111")

From add56aab3f83d1e18fc184839d187af49caf5ff2 Mon Sep 17 00:00:00 2001
From: claudiofr <claudiofr@yahoo.com>
Date: Mon, 23 Jan 2023 15:28:52 -0500
Subject: [PATCH 03/15] Fix testing errors after uploading changes to support a
 database based beaker cache.

Change test-requirements.txt to require v beaker 1.11.0 because latest
version of beaker apparently introduced a bug that broke the ability
to use database as a cache. Also fixed lint errors.
---
 lib/galaxy/config/__init__.py                 |  4 +---
 packages/tool_util/test-requirements.txt      |  3 ++-
 .../app/managers/test_CitationsManager_db.py  | 21 +++++++++++++++----
 .../tool_util/test_resolution_cache_db.py     |  2 +-
 4 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py
index 0ca4c5ed5eff..9bc573842d66 100644
--- a/lib/galaxy/config/__init__.py
+++ b/lib/galaxy/config/__init__.py
@@ -747,16 +747,14 @@ def _set_dependent_defaults(self):
         for dependent_config_param, config_param in DEPENDENT_CONFIG_DEFAULTS.items():
             try:
                 if getattr(self, dependent_config_param) is None:
-                    db_connect = getattr(self, config_param)
                     setattr(self, dependent_config_param, getattr(self, config_param))
             except AttributeError:
-                log.warning(
+                raise Exception(
                     "One or more invalid config parameter names specified in "
                     "DEPENDENT_CONFIG_DEFAULTS, "
                     f"{dependent_config_param}, {config_param}"
                 )
 
-
     def _configure_sqlalchemy20_warnings(self, kwargs):
         """
         This method should be deleted after migration to SQLAlchemy 2.0 is complete.
diff --git a/packages/tool_util/test-requirements.txt b/packages/tool_util/test-requirements.txt
index ebab0322ccf2..a9f13556b29f 100644
--- a/packages/tool_util/test-requirements.txt
+++ b/packages/tool_util/test-requirements.txt
@@ -1,3 +1,4 @@
-Beaker
+beaker
+sqlalchemy
 pytest
 pytest-mock
diff --git a/test/unit/app/managers/test_CitationsManager_db.py b/test/unit/app/managers/test_CitationsManager_db.py
index e2a8758d033c..50be2ec72839 100644
--- a/test/unit/app/managers/test_CitationsManager_db.py
+++ b/test/unit/app/managers/test_CitationsManager_db.py
@@ -1,14 +1,27 @@
-import os.path
-import tempfile
 import pytest
+from beaker.cache import CacheManager
+from beaker.util import parse_cache_config_options
 
 from galaxy.managers.citations import DoiCache
-from galaxy.util.bunch import Bunch
 from galaxy import config
 
+
+class TestDoiCache(DoiCache):
+    def __init__(self, config):
+        cache_opts = {
+            "cache.type": getattr(config, "citation_cache_type", "ext:database"),
+            "cache.data_dir": getattr(config, "citation_cache_data_dir", None),
+            "cache.lock_dir": getattr(config, "citation_cache_lock_dir", None),
+            "cache.url": "sqlite://",
+            "cache.table_name": getattr(config, "citation_cache_table_name", "citation_beaker_cache"),
+            "cache.schema_name": getattr(config, "citation_cache_schema_name", None),
+        }
+        self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi")
+
 @pytest.fixture()
 def doi_cache():
-    return DoiCache(config.GalaxyAppConfiguration(override_tempdir=False))
+    return TestDoiCache(config.GalaxyAppConfiguration(override_tempdir=False))
+
 
 def test_DoiCache(doi_cache):
     assert "Jörg" in doi_cache.get_bibtex("10.1093/bioinformatics/bts252")
diff --git a/test/unit/tool_util/test_resolution_cache_db.py b/test/unit/tool_util/test_resolution_cache_db.py
index 26d2d08a360f..e6ec9bcbb23e 100644
--- a/test/unit/tool_util/test_resolution_cache_db.py
+++ b/test/unit/tool_util/test_resolution_cache_db.py
@@ -25,7 +25,7 @@ def resolution_cache(tmpdir, appconfig):
         "cache.type": "ext:database",
         "cache.lock_dir": str(tmpdir / "lock"),
         "cache.expire": "1",
-        "cache.url": appconfig.mulled_resolution_cache_url,
+        "cache.url": "sqlite://",
         "cache.schema_name" : appconfig.mulled_resolution_cache_schema_name,
         "cache.table_name" : appconfig.mulled_resolution_cache_table_name,
     }

From 9237953e9a711635bbb47901653e436deb267f3b Mon Sep 17 00:00:00 2001
From: claudiofr <claudiofr@yahoo.com>
Date: Thu, 26 Jan 2023 15:52:35 -0500
Subject: [PATCH 04/15] Fix mypy errors after uploading changes to support a
 database based beaker cache.

mypy kept complaining that module 'galaxy' has no attribute 'config'
on an 'from galaxy import config' statement. So I changed it to
'import galaxy.config'
---
 test/unit/app/managers/test_CitationsManager_db.py | 5 +++--
 test/unit/tool_util/test_resolution_cache_db.py    | 6 ++++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/test/unit/app/managers/test_CitationsManager_db.py b/test/unit/app/managers/test_CitationsManager_db.py
index 50be2ec72839..46328174574d 100644
--- a/test/unit/app/managers/test_CitationsManager_db.py
+++ b/test/unit/app/managers/test_CitationsManager_db.py
@@ -3,7 +3,7 @@
 from beaker.util import parse_cache_config_options
 
 from galaxy.managers.citations import DoiCache
-from galaxy import config
+import galaxy.config
 
 
 class TestDoiCache(DoiCache):
@@ -18,9 +18,10 @@ def __init__(self, config):
         }
         self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi")
 
+
 @pytest.fixture()
 def doi_cache():
-    return TestDoiCache(config.GalaxyAppConfiguration(override_tempdir=False))
+    return TestDoiCache(galaxy.config.GalaxyAppConfiguration(override_tempdir=False))
 
 
 def test_DoiCache(doi_cache):
diff --git a/test/unit/tool_util/test_resolution_cache_db.py b/test/unit/tool_util/test_resolution_cache_db.py
index e6ec9bcbb23e..c7c4405c1654 100644
--- a/test/unit/tool_util/test_resolution_cache_db.py
+++ b/test/unit/tool_util/test_resolution_cache_db.py
@@ -4,6 +4,8 @@
 from beaker.cache import CacheManager
 from beaker.util import parse_cache_config_options
 
+#from galaxy import config
+import galaxy.config
 from galaxy.tool_util.deps.container_resolvers import ResolutionCache
 from galaxy.tool_util.deps.mulled.util import (
     _namespace_has_repo_name,
@@ -12,11 +14,11 @@
     TAG_CACHE_KEY,
 )
 
-from galaxy import config
 
 @pytest.fixture()
 def appconfig():
-    return config.GalaxyAppConfiguration(override_tempdir=False)
+    return galaxy.config.GalaxyAppConfiguration(override_tempdir=False)
+
 
 @pytest.fixture()
 def resolution_cache(tmpdir, appconfig):

From 1fa1b05011f52a4eb15d3153530d534b5e61f9a9 Mon Sep 17 00:00:00 2001
From: claudiofr <claudiofr@yahoo.com>
Date: Mon, 30 Jan 2023 18:01:20 -0500
Subject: [PATCH 05/15] Fix lint errors after uploading changes to support a
 database based beaker cache.

---
 test/unit/app/managers/test_CitationsManager_db.py | 4 ++--
 test/unit/tool_util/test_resolution_cache_db.py    | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/test/unit/app/managers/test_CitationsManager_db.py b/test/unit/app/managers/test_CitationsManager_db.py
index 46328174574d..d8514b8c5a36 100644
--- a/test/unit/app/managers/test_CitationsManager_db.py
+++ b/test/unit/app/managers/test_CitationsManager_db.py
@@ -6,7 +6,7 @@
 import galaxy.config
 
 
-class TestDoiCache(DoiCache):
+class MockDoiCache(DoiCache):
     def __init__(self, config):
         cache_opts = {
             "cache.type": getattr(config, "citation_cache_type", "ext:database"),
@@ -21,7 +21,7 @@ def __init__(self, config):
 
 @pytest.fixture()
 def doi_cache():
-    return TestDoiCache(galaxy.config.GalaxyAppConfiguration(override_tempdir=False))
+    return MockDoiCache(galaxy.config.GalaxyAppConfiguration(override_tempdir=False))
 
 
 def test_DoiCache(doi_cache):
diff --git a/test/unit/tool_util/test_resolution_cache_db.py b/test/unit/tool_util/test_resolution_cache_db.py
index c7c4405c1654..9deffdb03afa 100644
--- a/test/unit/tool_util/test_resolution_cache_db.py
+++ b/test/unit/tool_util/test_resolution_cache_db.py
@@ -4,7 +4,6 @@
 from beaker.cache import CacheManager
 from beaker.util import parse_cache_config_options
 
-#from galaxy import config
 import galaxy.config
 from galaxy.tool_util.deps.container_resolvers import ResolutionCache
 from galaxy.tool_util.deps.mulled.util import (

From 2d2ef309ad3312d15cea3367f60896cc6a09f370 Mon Sep 17 00:00:00 2001
From: claudiofr <claudiofr@yahoo.com>
Date: Tue, 31 Jan 2023 12:55:43 -0500
Subject: [PATCH 06/15] Fix formatting errors after uploading changes to
 support a database based beaker cache.

---
 test/unit/tool_util/test_resolution_cache_db.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/unit/tool_util/test_resolution_cache_db.py b/test/unit/tool_util/test_resolution_cache_db.py
index 9deffdb03afa..c6910f7137e2 100644
--- a/test/unit/tool_util/test_resolution_cache_db.py
+++ b/test/unit/tool_util/test_resolution_cache_db.py
@@ -27,8 +27,8 @@ def resolution_cache(tmpdir, appconfig):
         "cache.lock_dir": str(tmpdir / "lock"),
         "cache.expire": "1",
         "cache.url": "sqlite://",
-        "cache.schema_name" : appconfig.mulled_resolution_cache_schema_name,
-        "cache.table_name" : appconfig.mulled_resolution_cache_table_name,
+        "cache.schema_name": appconfig.mulled_resolution_cache_schema_name,
+        "cache.table_name": appconfig.mulled_resolution_cache_table_name,
     }
     cm = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("mulled_resolution")
     resolution_cache.mulled_resolution_cache = cm

From 57e9502f335c12802251aa194c4c53351a8c3033 Mon Sep 17 00:00:00 2001
From: claudiofr <claudiofr@yahoo.com>
Date: Tue, 31 Jan 2023 13:43:29 -0500
Subject: [PATCH 07/15] Fix isort errors after uploading changes to support a
 database based beaker cache.

---
 test/unit/app/managers/test_CitationsManager_db.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/unit/app/managers/test_CitationsManager_db.py b/test/unit/app/managers/test_CitationsManager_db.py
index d8514b8c5a36..7e7d66b59b54 100644
--- a/test/unit/app/managers/test_CitationsManager_db.py
+++ b/test/unit/app/managers/test_CitationsManager_db.py
@@ -2,8 +2,8 @@
 from beaker.cache import CacheManager
 from beaker.util import parse_cache_config_options
 
-from galaxy.managers.citations import DoiCache
 import galaxy.config
+from galaxy.managers.citations import DoiCache
 
 
 class MockDoiCache(DoiCache):

From cb9ff7c7b87de22e779af4e64f322b6e4e83e04a Mon Sep 17 00:00:00 2001
From: claudiofr <claudiofr@yahoo.com>
Date: Mon, 13 Feb 2023 15:02:22 -0500
Subject: [PATCH 08/15] Changed default beaker cache table name to beaker_cache
 for all caches

All 3 beaker caches, mulled_resolution, citations, biotools service,
now use the same default cache table, beaker_cache. This reduces the
number of open db connections because beaker creates a separate
sqlalchemy engine and associated connection pool for each distinct
table and each connection pool opens up a certain number of connections.
This was causing the integration tests to fail because the max
postgress connections(100) were being exceeded.
---
 doc/source/admin/galaxy_options.rst                | 6 +++---
 lib/galaxy/config/schemas/config_schema.yml        | 6 +++---
 lib/galaxy/managers/citations.py                   | 2 +-
 lib/galaxy/tools/biotools.py                       | 2 +-
 test/unit/app/managers/test_CitationsManager_db.py | 2 +-
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst
index 4e92fa0151a2..665c39662b49 100644
--- a/doc/source/admin/galaxy_options.rst
+++ b/doc/source/admin/galaxy_options.rst
@@ -1289,7 +1289,7 @@
     When biotools_service_cache_type = ext:database, this is
     the database table name used by beaker for
     bio.tools web service request related caching.
-:Default: ``biotools_service_beaker_cache``
+:Default: ``beaker_cache``
 :Type: str
 
 
@@ -1369,7 +1369,7 @@
     When citation_cache_type = ext:database, this is
     the database table name used by beaker for
     citation related caching.
-:Default: ``citation_beaker_cache``
+:Default: ``beaker_cache``
 :Type: str
 
 
@@ -1455,7 +1455,7 @@
     When mulled_resolution_cache_type = ext:database, this is
     the database table name used by beaker for
     caching mulled resolution requests. 
-:Default: ``mulled_resolution_beaker_cache``
+:Default: ``beaker_cache``
 :Type: str
 
 
diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml
index ef153d1ac8a5..8a3e0d397611 100644
--- a/lib/galaxy/config/schemas/config_schema.yml
+++ b/lib/galaxy/config/schemas/config_schema.yml
@@ -914,7 +914,7 @@ mapping:
 
       biotools_service_cache_table_name:
         type: str
-        default: biotools_service_beaker_cache
+        default: beaker_cache
         required: false
         desc: |
           Database table used by beaker for bio.tools web service request 
@@ -964,7 +964,7 @@ mapping:
 
       citation_cache_table_name:
         type: str
-        default: citation_beaker_cache
+        default: beaker_cache
         required: false
         desc: |
           Database table used by beaker for citation related caching.
@@ -1014,7 +1014,7 @@ mapping:
 
       mulled_resolution_cache_table_name:
         type: str
-        default: mulled_resolution_beaker_cache
+        default: beaker_cache
         required: false
         desc: |
           Database table used by beaker for caching mulled resolution requests.
diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py
index 451994fe93b3..cf88e5660603 100644
--- a/lib/galaxy/managers/citations.py
+++ b/lib/galaxy/managers/citations.py
@@ -42,7 +42,7 @@ def __init__(self, config):
             "cache.data_dir": getattr(config, "citation_cache_data_dir", None),
             "cache.lock_dir": getattr(config, "citation_cache_lock_dir", None),
             "cache.url": getattr(config, "citation_cache_url", None),
-            "cache.table_name": getattr(config, "citation_cache_table_name", "citation_beaker_cache"),
+            "cache.table_name": getattr(config, "citation_cache_table_name", None),
             "cache.schema_name": getattr(config, "citation_cache_schema_name", None),
         }
         self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi")
diff --git a/lib/galaxy/tools/biotools.py b/lib/galaxy/tools/biotools.py
index e8ce4d303bb7..3f8c1050cc8a 100644
--- a/lib/galaxy/tools/biotools.py
+++ b/lib/galaxy/tools/biotools.py
@@ -20,7 +20,7 @@ def get_galaxy_biotools_metadata_source(config) -> BiotoolsMetadataSource:
         "cache.data_dir": getattr(config, "biotools_service_cache_data_dir", None),
         "cache.lock_dir": getattr(config, "biotools_service_cache_lock_dir", None),
         "cache.url": getattr(config, "biotools_service_cache_url", config.database_connection),
-        "cache.table_name": getattr(config, "biotools_service_cache_table_name", "biotools_service_beaker_cache"),
+        "cache.table_name": getattr(config, "biotools_service_cache_table_name", None),
         "cache.schema_name": getattr(config, "biotools_service_cache_schema_name", None),
     }
     cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi")
diff --git a/test/unit/app/managers/test_CitationsManager_db.py b/test/unit/app/managers/test_CitationsManager_db.py
index 7e7d66b59b54..2615cbfd4621 100644
--- a/test/unit/app/managers/test_CitationsManager_db.py
+++ b/test/unit/app/managers/test_CitationsManager_db.py
@@ -13,7 +13,7 @@ def __init__(self, config):
             "cache.data_dir": getattr(config, "citation_cache_data_dir", None),
             "cache.lock_dir": getattr(config, "citation_cache_lock_dir", None),
             "cache.url": "sqlite://",
-            "cache.table_name": getattr(config, "citation_cache_table_name", "citation_beaker_cache"),
+            "cache.table_name": getattr(config, "citation_cache_table_name", None),
             "cache.schema_name": getattr(config, "citation_cache_schema_name", None),
         }
         self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi")

From 2d3547a630deb5b530cb0a45dcb656b0a3804ef3 Mon Sep 17 00:00:00 2001
From: claudiofr <claudiofr@yahoo.com>
Date: Sun, 19 Feb 2023 16:58:58 -0500
Subject: [PATCH 09/15] Change default beaker cache type to file for the 3
 beaker caches.

---
 doc/source/admin/galaxy_options.rst                | 6 +++---
 lib/galaxy/config/sample/galaxy.yml.sample         | 6 +++---
 lib/galaxy/config/schemas/config_schema.yml        | 6 +++---
 test/unit/app/managers/test_CitationsManager_db.py | 1 -
 test/unit/tool_util/test_resolution_cache_db.py    | 1 -
 5 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst
index 665c39662b49..574ae0f1eeeb 100644
--- a/doc/source/admin/galaxy_options.rst
+++ b/doc/source/admin/galaxy_options.rst
@@ -1225,7 +1225,7 @@
 :Description:
     bio.tools web service request related caching. The type of beaker
     cache used.
-:Default: ``ext:database``
+:Default: ``file``
 :Type: str
 
 
@@ -1302,7 +1302,7 @@
     fetched from external sources such as https://doi.org/ by Galaxy -
     the following parameters can be used to control the caching used
     to store this information.
-:Default: ``ext:database``
+:Default: ``file``
 :Type: str
 
 
@@ -1381,7 +1381,7 @@
     Mulled resolution caching. Mulled resolution uses external APIs of
     quay.io, these requests are caching using this and the following
     parameters
-:Default: ``ext:database``
+:Default: ``file``
 :Type: str
 
 
diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample
index a709bdbfb847..bdc3ae75b98c 100644
--- a/lib/galaxy/config/sample/galaxy.yml.sample
+++ b/lib/galaxy/config/sample/galaxy.yml.sample
@@ -924,7 +924,7 @@ galaxy:
 
   # bio.tools web service request related caching. The type of beaker
   # cache used.
-  #biotools_service_cache_type: ext:database
+  #biotools_service_cache_type: file
 
   # bio.tools web service request related caching. The data directory to
   # point beaker cache at.
@@ -959,7 +959,7 @@ galaxy:
   # from external sources such as https://doi.org/ by Galaxy - the
   # following parameters can be used to control the caching used to
   # store this information.
-  #citation_cache_type: ext:database
+  #citation_cache_type: file
 
   # Citation related caching.  Tool citations information maybe fetched
   # from external sources such as https://doi.org/ by Galaxy - the
@@ -997,7 +997,7 @@ galaxy:
   # Mulled resolution caching. Mulled resolution uses external APIs of
   # quay.io, these requests are caching using this and the following
   # parameters
-  #mulled_resolution_cache_type: ext:database
+  #mulled_resolution_cache_type: file
 
   # Data directory used by beaker for caching mulled resolution
   # requests.
diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml
index 8a3e0d397611..fbe3c5a376a0 100644
--- a/lib/galaxy/config/schemas/config_schema.yml
+++ b/lib/galaxy/config/schemas/config_schema.yml
@@ -882,7 +882,7 @@ mapping:
 
       biotools_service_cache_type:
         type: str
-        default: ext:database
+        default: file
         required: false
         desc: |
           bio.tools web service request related caching. The type of beaker cache used.
@@ -929,7 +929,7 @@ mapping:
 
       citation_cache_type:
         type: str
-        default: ext:database
+        default: file
         required: false
         desc: |
           Citation related caching.  Tool citations information maybe fetched from
@@ -977,7 +977,7 @@ mapping:
 
       mulled_resolution_cache_type:
         type: str
-        default: ext:database
+        default: file
         required: false
         desc: |
           Mulled resolution caching. Mulled resolution uses external APIs of quay.io, these
diff --git a/test/unit/app/managers/test_CitationsManager_db.py b/test/unit/app/managers/test_CitationsManager_db.py
index 2615cbfd4621..7812d9962161 100644
--- a/test/unit/app/managers/test_CitationsManager_db.py
+++ b/test/unit/app/managers/test_CitationsManager_db.py
@@ -11,7 +11,6 @@ def __init__(self, config):
         cache_opts = {
             "cache.type": getattr(config, "citation_cache_type", "ext:database"),
             "cache.data_dir": getattr(config, "citation_cache_data_dir", None),
-            "cache.lock_dir": getattr(config, "citation_cache_lock_dir", None),
             "cache.url": "sqlite://",
             "cache.table_name": getattr(config, "citation_cache_table_name", None),
             "cache.schema_name": getattr(config, "citation_cache_schema_name", None),
diff --git a/test/unit/tool_util/test_resolution_cache_db.py b/test/unit/tool_util/test_resolution_cache_db.py
index c6910f7137e2..4c448e329ac4 100644
--- a/test/unit/tool_util/test_resolution_cache_db.py
+++ b/test/unit/tool_util/test_resolution_cache_db.py
@@ -24,7 +24,6 @@ def resolution_cache(tmpdir, appconfig):
     resolution_cache = ResolutionCache()
     cache_opts = {
         "cache.type": "ext:database",
-        "cache.lock_dir": str(tmpdir / "lock"),
         "cache.expire": "1",
         "cache.url": "sqlite://",
         "cache.schema_name": appconfig.mulled_resolution_cache_schema_name,

From fe63ba9dcffcecb9c4b596d27edb8dff7779472b Mon Sep 17 00:00:00 2001
From: claudiofr <claudiofr@yahoo.com>
Date: Tue, 21 Feb 2023 15:43:50 -0500
Subject: [PATCH 10/15] Clear the beaker caches after creating them on app
 startup

Clear the caches in the event the cache type is database. This will
delete any rows in the cache table that could be left over from a
prior run of the app preventing an accumulation of stale data.
---
 lib/galaxy/app.py                             |  2 +
 lib/galaxy/managers/citations.py              | 14 ++++---
 .../unittest_utils/beaker_testing_utils.py    | 17 ++++++++
 lib/galaxy/tools/biotools.py                  | 14 ++++---
 .../app/managers/test_CitationsManager_db.py  | 38 +++++++++++-------
 .../tool_util/test_resolution_cache_db.py     | 40 +++++++++++++------
 6 files changed, 85 insertions(+), 40 deletions(-)
 create mode 100644 lib/galaxy/model/unittest_utils/beaker_testing_utils.py

diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py
index 767cc27bbd07..7fbc2ac43fae 100644
--- a/lib/galaxy/app.py
+++ b/lib/galaxy/app.py
@@ -330,6 +330,8 @@ def _configure_toolbox(self):
             mulled_resolution_cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache(
                 "mulled_resolution"
             )
+            # If using database cache clear cache table contents
+            mulled_resolution_cache.clear()
         self.container_finder = containers.ContainerFinder(app_info, mulled_resolution_cache=mulled_resolution_cache)
         self._set_enabled_container_types()
         index_help = getattr(self.config, "index_tool_help", True)
diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py
index cf88e5660603..c5c5c36a8538 100644
--- a/lib/galaxy/managers/citations.py
+++ b/lib/galaxy/managers/citations.py
@@ -38,14 +38,16 @@ def _get_tool(self, tool_id):
 class DoiCache:
     def __init__(self, config):
         cache_opts = {
-            "cache.type": getattr(config, "citation_cache_type", "ext:database"),
-            "cache.data_dir": getattr(config, "citation_cache_data_dir", None),
-            "cache.lock_dir": getattr(config, "citation_cache_lock_dir", None),
-            "cache.url": getattr(config, "citation_cache_url", None),
-            "cache.table_name": getattr(config, "citation_cache_table_name", None),
-            "cache.schema_name": getattr(config, "citation_cache_schema_name", None),
+            "cache.type": config.citation_cache_type,
+            "cache.data_dir": config.citation_cache_data_dir,
+            "cache.lock_dir": config.citation_cache_lock_dir,
+            "cache.url": config.citation_cache_url,
+            "cache.table_name": config.citation_cache_table_name,
+            "cache.schema_name": config.citation_cache_schema_name,
         }
         self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi")
+        # If using database cache clear cache table contents
+        self._cache.clear()
 
     def _raw_get_bibtex(self, doi):
         doi_url = f"https://doi.org/{doi}"
diff --git a/lib/galaxy/model/unittest_utils/beaker_testing_utils.py b/lib/galaxy/model/unittest_utils/beaker_testing_utils.py
new file mode 100644
index 000000000000..1bfd2bf7f7c3
--- /dev/null
+++ b/lib/galaxy/model/unittest_utils/beaker_testing_utils.py
@@ -0,0 +1,17 @@
+from sqlalchemy import (
+    MetaData,
+    select,
+    Table,
+)
+
+from galaxy.model.unittest_utils.model_testing_utils import disposing_engine
+
+
+def is_cache_empty(url: str, namespace: str, beaker_table: str = "beaker_cache") -> bool:
+    """Check if there are any entries for a given namespace in a beaker cache db table"""
+    with disposing_engine(url) as eng:
+        metadata_obj = MetaData()
+        table = Table(beaker_table, metadata_obj, autoload_with=eng)
+        with eng.connect() as conn:
+            result = conn.execute(select(table).where(table.c.namespace == namespace))
+            return result.fetchone() is None
diff --git a/lib/galaxy/tools/biotools.py b/lib/galaxy/tools/biotools.py
index 3f8c1050cc8a..d2c0a82738f9 100644
--- a/lib/galaxy/tools/biotools.py
+++ b/lib/galaxy/tools/biotools.py
@@ -16,13 +16,15 @@ def get_galaxy_biotools_metadata_source(config) -> BiotoolsMetadataSource:
     biotools_metadata_source_config.content_directory = config.biotools_content_directory
     biotools_metadata_source_config.use_api = config.biotools_use_api
     cache_opts = {
-        "cache.type": getattr(config, "biotools_service_cache_type", "ext:database"),
-        "cache.data_dir": getattr(config, "biotools_service_cache_data_dir", None),
-        "cache.lock_dir": getattr(config, "biotools_service_cache_lock_dir", None),
-        "cache.url": getattr(config, "biotools_service_cache_url", config.database_connection),
-        "cache.table_name": getattr(config, "biotools_service_cache_table_name", None),
-        "cache.schema_name": getattr(config, "biotools_service_cache_schema_name", None),
+        "cache.type": config.biotools_service_cache_type,
+        "cache.data_dir": config.biotools_service_cache_data_dir,
+        "cache.lock_dir": config.biotools_service_cache_lock_dir,
+        "cache.url": config.biotools_service_cache_url,
+        "cache.table_name": config.biotools_service_cache_table_name,
+        "cache.schema_name": config.biotools_service_cache_schema_name,
     }
     cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi")
+    # If using database cache clear cache table contents
+    cache.clear()
     biotools_metadata_source_config.cache = cache
     return get_biotools_metadata_source(biotools_metadata_source_config)
diff --git a/test/unit/app/managers/test_CitationsManager_db.py b/test/unit/app/managers/test_CitationsManager_db.py
index 7812d9962161..8c7b8f8eb29c 100644
--- a/test/unit/app/managers/test_CitationsManager_db.py
+++ b/test/unit/app/managers/test_CitationsManager_db.py
@@ -1,28 +1,36 @@
-import pytest
 from beaker.cache import CacheManager
 from beaker.util import parse_cache_config_options
 
 import galaxy.config
 from galaxy.managers.citations import DoiCache
+from galaxy.model.unittest_utils.beaker_testing_utils import is_cache_empty
+from galaxy.model.unittest_utils.migration_scripts_testing_utils import tmp_directory  # noqa: F401
+from galaxy.model.unittest_utils.model_testing_utils import (  # noqa: F401  (url_factory is a fixture we have to import explicitly)
+    create_and_drop_database,
+    url_factory,
+)
 
 
 class MockDoiCache(DoiCache):
-    def __init__(self, config):
+    def __init__(self, config, db_url):
         cache_opts = {
-            "cache.type": getattr(config, "citation_cache_type", "ext:database"),
-            "cache.data_dir": getattr(config, "citation_cache_data_dir", None),
-            "cache.url": "sqlite://",
-            "cache.table_name": getattr(config, "citation_cache_table_name", None),
-            "cache.schema_name": getattr(config, "citation_cache_schema_name", None),
+            "cache.type": "ext:database",
+            "cache.data_dir": config.citation_cache_data_dir,
+            "cache.url": db_url,
+            "cache.table_name": config.citation_cache_table_name,
+            "cache.schema_name": config.citation_cache_schema_name,
         }
         self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi")
+        self._cache.clear()
 
 
-@pytest.fixture()
-def doi_cache():
-    return MockDoiCache(galaxy.config.GalaxyAppConfiguration(override_tempdir=False))
-
-
-def test_DoiCache(doi_cache):
-    assert "Jörg" in doi_cache.get_bibtex("10.1093/bioinformatics/bts252")
-    assert "Özkurt" in doi_cache.get_bibtex("10.1101/2021.12.24.474111")
+def test_DoiCache(url_factory):  # noqa: F811
+    db_url = url_factory()
+    with create_and_drop_database(db_url):
+        doi_cache = MockDoiCache(galaxy.config.GalaxyAppConfiguration(override_tempdir=False), db_url)
+        assert is_cache_empty(db_url, "doi")
+        assert "Jörg" in doi_cache.get_bibtex("10.1093/bioinformatics/bts252")
+        assert "Özkurt" in doi_cache.get_bibtex("10.1101/2021.12.24.474111")
+        assert not is_cache_empty(db_url, "doi")
+        doi_cache._cache.clear()
+        assert is_cache_empty(db_url, "doi")
diff --git a/test/unit/tool_util/test_resolution_cache_db.py b/test/unit/tool_util/test_resolution_cache_db.py
index 4c448e329ac4..156fbbc7c8a7 100644
--- a/test/unit/tool_util/test_resolution_cache_db.py
+++ b/test/unit/tool_util/test_resolution_cache_db.py
@@ -5,6 +5,12 @@
 from beaker.util import parse_cache_config_options
 
 import galaxy.config
+from galaxy.model.unittest_utils.beaker_testing_utils import is_cache_empty
+from galaxy.model.unittest_utils.migration_scripts_testing_utils import tmp_directory  # noqa: F401
+from galaxy.model.unittest_utils.model_testing_utils import (  # noqa: F401  (url_factory is a fixture we have to import explicitly)
+    create_and_drop_database,
+    url_factory,
+)
 from galaxy.tool_util.deps.container_resolvers import ResolutionCache
 from galaxy.tool_util.deps.mulled.util import (
     _namespace_has_repo_name,
@@ -13,25 +19,33 @@
     TAG_CACHE_KEY,
 )
 
+cache_namespace = "mulled_resolution"
 
-@pytest.fixture()
+
+@pytest.fixture(scope="module")
 def appconfig():
     return galaxy.config.GalaxyAppConfiguration(override_tempdir=False)
 
 
 @pytest.fixture()
-def resolution_cache(tmpdir, appconfig):
-    resolution_cache = ResolutionCache()
-    cache_opts = {
-        "cache.type": "ext:database",
-        "cache.expire": "1",
-        "cache.url": "sqlite://",
-        "cache.schema_name": appconfig.mulled_resolution_cache_schema_name,
-        "cache.table_name": appconfig.mulled_resolution_cache_table_name,
-    }
-    cm = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("mulled_resolution")
-    resolution_cache.mulled_resolution_cache = cm
-    return resolution_cache
+def resolution_cache(url_factory, appconfig):  # noqa: F811
+    db_url = url_factory()
+    with create_and_drop_database(db_url):
+        resolution_cache = ResolutionCache()
+        cache_opts = {
+            "cache.type": "ext:database",
+            "cache.expire": "1",
+            "cache.url": db_url,
+            "cache.schema_name": appconfig.mulled_resolution_cache_schema_name,
+            "cache.table_name": appconfig.mulled_resolution_cache_table_name,
+        }
+        cm = CacheManager(**parse_cache_config_options(cache_opts)).get_cache(cache_namespace)
+        cm.clear()
+        resolution_cache.mulled_resolution_cache = cm
+        yield resolution_cache
+        assert not is_cache_empty(db_url, cache_namespace)
+        cm.clear()
+        assert is_cache_empty(db_url, cache_namespace)
 
 
 def test_resolution_cache_namepace_has_repo_name(resolution_cache):

From 17f22c28f3cdc7825b4e150f1d3ee33aeb8ed12c Mon Sep 17 00:00:00 2001
From: claudiofr <claudiofr@yahoo.com>
Date: Tue, 28 Feb 2023 09:15:25 -0500
Subject: [PATCH 11/15] Add new config parameters for citation database beaker
 cache in tool_shed yaml file

---
 lib/galaxy/app.py                             |  2 --
 lib/galaxy/config/sample/tool_shed.yml.sample | 17 +++++++++++++++++
 .../schemas/tool_shed_config_schema.yml       | 19 +++++++++++++++++++
 lib/galaxy/managers/citations.py              |  2 --
 .../unittest_utils/beaker_testing_utils.py    |  2 +-
 lib/galaxy/tools/biotools.py                  |  2 --
 lib/tool_shed/webapp/config.py                |  3 +++
 packages/tool_util/test-requirements.txt      |  3 +--
 .../app/managers/test_CitationsManager.py     |  3 +++
 9 files changed, 44 insertions(+), 9 deletions(-)

diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py
index 7fbc2ac43fae..767cc27bbd07 100644
--- a/lib/galaxy/app.py
+++ b/lib/galaxy/app.py
@@ -330,8 +330,6 @@ def _configure_toolbox(self):
             mulled_resolution_cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache(
                 "mulled_resolution"
             )
-            # If using database cache clear cache table contents
-            mulled_resolution_cache.clear()
         self.container_finder = containers.ContainerFinder(app_info, mulled_resolution_cache=mulled_resolution_cache)
         self._set_enabled_container_types()
         index_help = getattr(self.config, "index_tool_help", True)
diff --git a/lib/galaxy/config/sample/tool_shed.yml.sample b/lib/galaxy/config/sample/tool_shed.yml.sample
index 9744f777c9d5..5f08dfbf430e 100644
--- a/lib/galaxy/config/sample/tool_shed.yml.sample
+++ b/lib/galaxy/config/sample/tool_shed.yml.sample
@@ -288,6 +288,23 @@ tool_shed:
   # store this information.
   #citation_cache_lock_dir: database/citations/lock
 
+  # When citation_cache_type = ext:database, this is
+  # the url of the database used by beaker for citation
+  # related caching.
+  # The application config code will set it to the 
+  # value of database_connection if this is not set.
+  #citation_cache_url: null
+
+  # When citation_cache_type = ext:database, this is
+  # the database schema name of the table used by beaker
+  # for citation related caching.
+  #citation_cache_schema_name: null
+
+  # When citation_cache_type = ext:database, this is
+  # the database table name used by beaker
+  # for citation related caching.
+  #citation_cache_table_name: null
+
   # Turn on logging of user actions to the database.  Actions currently
   # logged are grid views, tool searches, and use of "recently" used
   # tools menu.  The log_events and log_actions functionality will
diff --git a/lib/galaxy/config/schemas/tool_shed_config_schema.yml b/lib/galaxy/config/schemas/tool_shed_config_schema.yml
index 37612b580713..116f0521f483 100644
--- a/lib/galaxy/config/schemas/tool_shed_config_schema.yml
+++ b/lib/galaxy/config/schemas/tool_shed_config_schema.yml
@@ -520,6 +520,25 @@ mapping:
           external sources such as https://doi.org/ by Galaxy - the following
           parameters can be used to control the caching used to store this information.
 
+      citation_cache_url:
+        type: str
+        required: false
+        desc: |
+          Url to database used by beaker for citation related caching.
+
+      citation_cache_table_name:
+        type: str
+        default: beaker_cache
+        required: false
+        desc: |
+          Database table used by beaker for citation related caching.
+
+      citation_cache_schema_name:
+        type: str
+        required: false
+        desc: |
+          Schema of database table used by beaker for citation related caching.
+
       log_actions:
         type: bool
         default: false
diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py
index c5c5c36a8538..64e8ca91754d 100644
--- a/lib/galaxy/managers/citations.py
+++ b/lib/galaxy/managers/citations.py
@@ -46,8 +46,6 @@ def __init__(self, config):
             "cache.schema_name": config.citation_cache_schema_name,
         }
         self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi")
-        # If using database cache clear cache table contents
-        self._cache.clear()
 
     def _raw_get_bibtex(self, doi):
         doi_url = f"https://doi.org/{doi}"
diff --git a/lib/galaxy/model/unittest_utils/beaker_testing_utils.py b/lib/galaxy/model/unittest_utils/beaker_testing_utils.py
index 1bfd2bf7f7c3..d2480aa4f969 100644
--- a/lib/galaxy/model/unittest_utils/beaker_testing_utils.py
+++ b/lib/galaxy/model/unittest_utils/beaker_testing_utils.py
@@ -9,7 +9,7 @@
 
 def is_cache_empty(url: str, namespace: str, beaker_table: str = "beaker_cache") -> bool:
     """Check if there are any entries for a given namespace in a beaker cache db table"""
-    with disposing_engine(url) as eng:
+    with disposing_engine(url) as eng:  # type: ignore[arg-type]
         metadata_obj = MetaData()
         table = Table(beaker_table, metadata_obj, autoload_with=eng)
         with eng.connect() as conn:
diff --git a/lib/galaxy/tools/biotools.py b/lib/galaxy/tools/biotools.py
index d2c0a82738f9..a27fe7afef34 100644
--- a/lib/galaxy/tools/biotools.py
+++ b/lib/galaxy/tools/biotools.py
@@ -24,7 +24,5 @@ def get_galaxy_biotools_metadata_source(config) -> BiotoolsMetadataSource:
         "cache.schema_name": config.biotools_service_cache_schema_name,
     }
     cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi")
-    # If using database cache clear cache table contents
-    cache.clear()
     biotools_metadata_source_config.cache = cache
     return get_biotools_metadata_source(biotools_metadata_source_config)
diff --git a/lib/tool_shed/webapp/config.py b/lib/tool_shed/webapp/config.py
index ec22c7ff514f..4511b54e1cd7 100644
--- a/lib/tool_shed/webapp/config.py
+++ b/lib/tool_shed/webapp/config.py
@@ -130,6 +130,9 @@ def _process_config(self, kwargs):
         self.citation_cache_lock_dir = self._in_root_dir(
             kwargs.get("citation_cache_lock_dir", "database/tool_shed_citations/locks")
         )
+        self.citation_cache_url = kwargs.get("citation_cache_lock_dir", None)
+        self.citation_cache_schema_name = kwargs.get("citation_cache_schema_name", None)
+        self.citation_cache_table_name = kwargs.get("citation_cache_table_name", None)
         self.password_expiration_period = timedelta(days=int(self.password_expiration_period))
 
         # Security/Policy Compliance
diff --git a/packages/tool_util/test-requirements.txt b/packages/tool_util/test-requirements.txt
index a9f13556b29f..cb75bd1ec03e 100644
--- a/packages/tool_util/test-requirements.txt
+++ b/packages/tool_util/test-requirements.txt
@@ -1,4 +1,3 @@
-beaker
-sqlalchemy
+beaker==1.11.0 ; python_version >= "3.7" and python_version < "3.12"
 pytest
 pytest-mock
diff --git a/test/unit/app/managers/test_CitationsManager.py b/test/unit/app/managers/test_CitationsManager.py
index 030fc6de489c..027ab0530380 100644
--- a/test/unit/app/managers/test_CitationsManager.py
+++ b/test/unit/app/managers/test_CitationsManager.py
@@ -11,6 +11,9 @@ def test_DoiCache():
             citation_cache_type="file",
             citation_cache_data_dir=os.path.join(tmp_database_dir, "data"),
             citation_cache_lock_dir=os.path.join(tmp_database_dir, "locks"),
+            citation_cache_url=None,
+            citation_cache_table_name=None,
+            citation_cache_schema_name=None,
         )
         doi_cache = DoiCache(config)
         assert "Jörg" in doi_cache.get_bibtex("10.1093/bioinformatics/bts252")

From ae9746207311fb24b2c35c101b3c3ba2ef27d62c Mon Sep 17 00:00:00 2001
From: claudiofr <claudiofr@yahoo.com>
Date: Tue, 28 Feb 2023 09:38:00 -0500
Subject: [PATCH 12/15] Move test_resolution_cache_db.py to test/unit/app/tools
 directory

The test/unit/app directory tree has a test-requirements.txt file that
brings in dependencies required by database utility modules imported by
the unit test.
---
 test/unit/{tool_util => app/tools}/test_resolution_cache_db.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename test/unit/{tool_util => app/tools}/test_resolution_cache_db.py (100%)

diff --git a/test/unit/tool_util/test_resolution_cache_db.py b/test/unit/app/tools/test_resolution_cache_db.py
similarity index 100%
rename from test/unit/tool_util/test_resolution_cache_db.py
rename to test/unit/app/tools/test_resolution_cache_db.py

From 3d186cbb0056062bfb8c8c3a3e681ce2a0e2d26a Mon Sep 17 00:00:00 2001
From: Marius van den Beek <m.vandenbeek@gmail.com>
Date: Mon, 6 Mar 2023 11:30:21 +0100
Subject: [PATCH 13/15] Drop beaker pin

Co-authored-by: Nicola Soranzo <nicola.soranzo@gmail.com>
---
 packages/tool_util/test-requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/tool_util/test-requirements.txt b/packages/tool_util/test-requirements.txt
index cb75bd1ec03e..ebab0322ccf2 100644
--- a/packages/tool_util/test-requirements.txt
+++ b/packages/tool_util/test-requirements.txt
@@ -1,3 +1,3 @@
-beaker==1.11.0 ; python_version >= "3.7" and python_version < "3.12"
+Beaker
 pytest
 pytest-mock

From f096dbf18a686e321a6328241aedb3bed710da83 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Wed, 21 Jun 2023 22:43:16 +0200
Subject: [PATCH 14/15] Update galaxy config schema with newer version of
 options

---
 lib/galaxy/config/schemas/config_schema.yml | 43 +++++++++++++++------
 1 file changed, 31 insertions(+), 12 deletions(-)

diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml
index fbe3c5a376a0..175e792996d8 100644
--- a/lib/galaxy/config/schemas/config_schema.yml
+++ b/lib/galaxy/config/schemas/config_schema.yml
@@ -909,23 +909,28 @@ mapping:
         type: str
         required: false
         desc: |
-          Url to database used by beaker for bio.tools web service request 
-          related caching.
+          When biotools_service_cache_type = ext:database, this is
+          the url of the database used by beaker for
+          bio.tools web service request related caching.
+          The application config code will set it to the
+          value of database_connection if this is not set.
 
       biotools_service_cache_table_name:
         type: str
         default: beaker_cache
         required: false
         desc: |
-          Database table used by beaker for bio.tools web service request 
-          related caching.
+          When biotools_service_cache_type = ext:database, this is
+          the database table name used by beaker for
+          bio.tools web service request related caching.
 
       biotools_service_cache_schema_name:
         type: str
         required: false
         desc: |
-          Schema of database table used by beaker for bio.tools web service request 
-          related caching.
+          When biotools_service_cache_type = ext:database, this is
+          the database table name used by beaker for
+          bio.tools web service request related caching.
 
       citation_cache_type:
         type: str
@@ -960,20 +965,27 @@ mapping:
         type: str
         required: false
         desc: |
-          Url to database used by beaker for citation related caching.
+          When citation_cache_type = ext:database, this is
+          the url of the database used by beaker for citation
+          caching. The application config code will set it to the
+          value of database_connection if this is not set.
 
       citation_cache_table_name:
         type: str
         default: beaker_cache
         required: false
         desc: |
-          Database table used by beaker for citation related caching.
+          When citation_cache_type = ext:database, this is
+          the database table name used by beaker for
+          citation related caching.
 
       citation_cache_schema_name:
         type: str
         required: false
         desc: |
-          Schema of database table used by beaker for citation related caching.
+          When citation_cache_type = ext:database, this is
+          the database schema name of the table used by beaker for
+          citation related caching.
 
       mulled_resolution_cache_type:
         type: str
@@ -1010,20 +1022,27 @@ mapping:
         type: str
         required: false
         desc: |
-          Url to database used by beaker for caching mulled resolution requests.
+          When mulled_resolution_cache_type = ext:database, this is
+          the url of the database used by beaker for caching mulled resolution
+          requests. The application config code will set it to the
+          value of database_connection if this is not set.
 
       mulled_resolution_cache_table_name:
         type: str
         default: beaker_cache
         required: false
         desc: |
-          Database table used by beaker for caching mulled resolution requests.
+          When mulled_resolution_cache_type = ext:database, this is
+          the database table name used by beaker for
+          caching mulled resolution requests.
 
       mulled_resolution_cache_schema_name:
         type: str
         required: false
         desc: |
-          Schema of database table used by beaker for caching mulled resolution requests.
+          When mulled_resolution_cache_type = ext:database, this is
+          the database schema name of the table used by beaker for
+          caching mulled resolution requests.
 
       object_store_config_file:
         type: str

From 3606c34540869321f959163c818a428da51de945 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Wed, 21 Jun 2023 22:43:50 +0200
Subject: [PATCH 15/15] Rebuild schema docs

---
 doc/source/admin/galaxy_options.rst           | 169 ++++++++++++------
 lib/galaxy/config/sample/galaxy.yml.sample    | 102 +++++++----
 lib/galaxy/config/sample/tool_shed.yml.sample |  19 +-
 3 files changed, 185 insertions(+), 105 deletions(-)

diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst
index 574ae0f1eeeb..3ce3c4ce53c7 100644
--- a/doc/source/admin/galaxy_options.rst
+++ b/doc/source/admin/galaxy_options.rst
@@ -773,7 +773,7 @@
     container resolvers to use when discovering containers for Galaxy.
     If this is set to None, the default container resolvers loaded is
     determined by enable_mulled_containers. For available options see
-    config/container_resolvers_conf.xml.sample.
+    https://docs.galaxyproject.org/en/master/admin/container_resolvers.html
 :Default: ``None``
 :Type: str
 
@@ -1260,36 +1260,35 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 :Description:
-    When biotools_service_cache_type = ext:database, this is
-    the url of the database used by beaker for
-    bio.tools web service request related caching.
-    The application config code will set it to the 
+    When biotools_service_cache_type = ext:database, this is the url
+    of the database used by beaker for bio.tools web service request
+    related caching. The application config code will set it to the
     value of database_connection if this is not set.
 :Default: ``None``
 :Type: str
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-``biotools_service_cache_schema_name``
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``biotools_service_cache_table_name``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 :Description:
-    When biotools_service_cache_type = ext:database, this is
-    the database schema name of the table used by beaker for
-    bio.tools web service request related caching.
-:Default: ``None``
+    When biotools_service_cache_type = ext:database, this is the
+    database table name used by beaker for bio.tools web service
+    request related caching.
+:Default: ``beaker_cache``
 :Type: str
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-``biotools_service_cache_table_name``
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``biotools_service_cache_schema_name``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 :Description:
-    When biotools_service_cache_type = ext:database, this is
-    the database table name used by beaker for
-    bio.tools web service request related caching.
-:Default: ``beaker_cache``
+    When biotools_service_cache_type = ext:database, this is the
+    database table name used by beaker for bio.tools web service
+    request related caching.
+:Default: ``None``
 :Type: str
 
 
@@ -1341,35 +1340,34 @@
 ~~~~~~~~~~~~~~~~~~~~~~
 
 :Description:
-    When citation_cache_type = ext:database, this is
-    the url of the database used by beaker for citation
-    caching. The application config code will set it to the 
-    value of database_connection if this is not set.
+    When citation_cache_type = ext:database, this is the url of the
+    database used by beaker for citation caching. The application
+    config code will set it to the value of database_connection if
+    this is not set.
 :Default: ``None``
 :Type: str
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-``citation_cache_schema_name``
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``citation_cache_table_name``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 :Description:
-    When citation_cache_type = ext:database, this is
-    the database schema name of the table used by beaker for
-    citation related caching.
-:Default: ``None``
+    When citation_cache_type = ext:database, this is the database
+    table name used by beaker for citation related caching.
+:Default: ``beaker_cache``
 :Type: str
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-``citation_cache_table_name``
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``citation_cache_schema_name``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 :Description:
-    When citation_cache_type = ext:database, this is
-    the database table name used by beaker for
-    citation related caching.
-:Default: ``beaker_cache``
+    When citation_cache_type = ext:database, this is the database
+    schema name of the table used by beaker for citation related
+    caching.
+:Default: ``None``
 :Type: str
 
 
@@ -1427,35 +1425,35 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 :Description:
-    When mulled_resolution_cache_type = ext:database, this is
-    the url of the database used by beaker for caching mulled resolution
-    requests. The application config code will set it to the 
-    value of database_connection if this is not set.
+    When mulled_resolution_cache_type = ext:database, this is the url
+    of the database used by beaker for caching mulled resolution
+    requests. The application config code will set it to the value of
+    database_connection if this is not set.
 :Default: ``None``
 :Type: str
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-``mulled_resolution_cache_schema_name``
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``mulled_resolution_cache_table_name``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 :Description:
-    When mulled_resolution_cache_type = ext:database, this is
-    the database schema name of the table used by beaker for
-    caching mulled resolution requests. 
-:Default: ``None``
+    When mulled_resolution_cache_type = ext:database, this is the
+    database table name used by beaker for caching mulled resolution
+    requests.
+:Default: ``beaker_cache``
 :Type: str
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-``mulled_resolution_cache_table_name``
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``mulled_resolution_cache_schema_name``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 :Description:
-    When mulled_resolution_cache_type = ext:database, this is
-    the database table name used by beaker for
-    caching mulled resolution requests. 
-:Default: ``beaker_cache``
+    When mulled_resolution_cache_type = ext:database, this is the
+    database schema name of the table used by beaker for caching
+    mulled resolution requests.
+:Default: ``None``
 :Type: str
 
 
@@ -1472,6 +1470,67 @@
 :Type: str
 
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``object_store_cache_monitor_driver``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Specify where cache monitoring is driven for caching object stores
+    such as S3, Azure, and iRODS. This option has no affect on disk
+    object stores. For production instances, the cache should be
+    monitored by external tools such as tmpwatch and this value should
+    be set to 'external'. This will disable all cache monitoring in
+    Galaxy. Alternatively, 'celery' can monitor caches using a
+    periodic task or an 'inprocess' thread can be used - but this last
+    option seriously limits Galaxy's ability to scale. The default of
+    'auto' will use 'celery' if 'enable_celery_tasks' is set to true
+    or 'inprocess' otherwise. This option serves as the default for
+    all object stores and can be overridden on a per object store
+    basis (but don't - just setup tmpwatch for all relevant cache
+    paths).
+:Default: ``auto``
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``object_store_cache_monitor_interval``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    For object store cache monitoring done by Galaxy, this is the
+    interval between cache checking steps. This is used by both
+    inprocess cache monitors (which we recommend you do not use) and
+    by the celery task if it is configured (by setting
+    enable_celery_tasks to true and not setting
+    object_store_cache_monitor_driver to external).
+:Default: ``600``
+:Type: int
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``object_store_cache_path``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Default cache path for caching object stores if cache not
+    configured for that object store entry.
+    The value of this option will be resolved with respect to
+    <cache_dir>.
+:Default: ``object_store_cache``
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``object_store_cache_size``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Default cache size for caching object stores if cache not
+    configured for that object store entry.
+:Default: ``-1``
+:Type: int
+
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 ``object_store_store_by``
 ~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample
index bdc3ae75b98c..506300418a4f 100644
--- a/lib/galaxy/config/sample/galaxy.yml.sample
+++ b/lib/galaxy/config/sample/galaxy.yml.sample
@@ -938,22 +938,21 @@ galaxy:
   # <cache_dir>.
   #biotools_service_cache_lock_dir: biotools/locks
 
-  # When biotools_service_cache_type = ext:database, this is
-  # the url of the database used by beaker
-  # for bio.tools web service request caching.
-  # The application config code will set it to the 
+  # When biotools_service_cache_type = ext:database, this is the url of
+  # the database used by beaker for bio.tools web service request
+  # related caching. The application config code will set it to the
   # value of database_connection if this is not set.
   #biotools_service_cache_url: null
 
-  # When biotools_service_cache_type = ext:database, this is
-  # the database schema name of the table used by beaker
-  # for bio.tools web service request caching.
-  #biotools_service_cache_schema_name: null
+  # When biotools_service_cache_type = ext:database, this is the
+  # database table name used by beaker for bio.tools web service request
+  # related caching.
+  #biotools_service_cache_table_name: beaker_cache
 
-  # When biotools_service_cache_type = ext:database, this is
-  # the database table name used by beaker
-  # for bio.tools web service request caching.
-  #biotools_service_cache_table_name: null
+  # When biotools_service_cache_type = ext:database, this is the
+  # database table name used by beaker for bio.tools web service request
+  # related caching.
+  #biotools_service_cache_schema_name: null
 
   # Citation related caching.  Tool citations information maybe fetched
   # from external sources such as https://doi.org/ by Galaxy - the
@@ -977,22 +976,19 @@ galaxy:
   # <cache_dir>.
   #citation_cache_lock_dir: citations/locks
 
-  # When citation_cache_type = ext:database, this is
-  # the url of the database used by beaker for citation
-  # related caching.
-  # The application config code will set it to the 
-  # value of database_connection if this is not set.
+  # When citation_cache_type = ext:database, this is the url of the
+  # database used by beaker for citation caching. The application config
+  # code will set it to the value of database_connection if this is not
+  # set.
   #citation_cache_url: null
 
-  # When citation_cache_type = ext:database, this is
-  # the database schema name of the table used by beaker
-  # for citation related caching.
-  #citation_cache_schema_name: null
+  # When citation_cache_type = ext:database, this is the database table
+  # name used by beaker for citation related caching.
+  #citation_cache_table_name: beaker_cache
 
-  # When citation_cache_type = ext:database, this is
-  # the database table name used by beaker
-  # for citation related caching.
-  #citation_cache_table_name: null
+  # When citation_cache_type = ext:database, this is the database schema
+  # name of the table used by beaker for citation related caching.
+  #citation_cache_schema_name: null
 
   # Mulled resolution caching. Mulled resolution uses external APIs of
   # quay.io, these requests are caching using this and the following
@@ -1015,21 +1011,21 @@ galaxy:
   # created.
   #mulled_resolution_cache_expire: 3600
 
-  # When mulled_resolution_cache_type = ext:database, this is
-  # the url of the database used by beaker for caching mulled resolution
-  # requests. The application config code will set it to the 
-  # value of database_connection if this is not set.
+  # When mulled_resolution_cache_type = ext:database, this is the url of
+  # the database used by beaker for caching mulled resolution requests.
+  # The application config code will set it to the value of
+  # database_connection if this is not set.
   #mulled_resolution_cache_url: null
 
-  # When mulled_resolution_cache_type = ext:database, this is
-  # the database schema name of the table used by beaker for
-  # caching mulled resolution requests. 
-  #mulled_resolution_cache_schema_name: null
+  # When mulled_resolution_cache_type = ext:database, this is the
+  # database table name used by beaker for caching mulled resolution
+  # requests.
+  #mulled_resolution_cache_table_name: beaker_cache
 
-  # When mulled_resolution_cache_type = ext:database, this is
-  # the database table name used by beaker for
-  # caching mulled resolution requests. 
-  #mulled_resolution_cache_table_name: null
+  # When mulled_resolution_cache_type = ext:database, this is the
+  # database schema name of the table used by beaker for caching mulled
+  # resolution requests.
+  #mulled_resolution_cache_schema_name: null
 
   # Configuration file for the object store If this is set and exists,
   # it overrides any other objectstore settings.
@@ -1037,6 +1033,38 @@ galaxy:
   # <config_dir>.
   #object_store_config_file: object_store_conf.xml
 
+  # Specify where cache monitoring is driven for caching object stores
+  # such as S3, Azure, and iRODS. This option has no affect on disk
+  # object stores. For production instances, the cache should be
+  # monitored by external tools such as tmpwatch and this value should
+  # be set to 'external'. This will disable all cache monitoring in
+  # Galaxy. Alternatively, 'celery' can monitor caches using a periodic
+  # task or an 'inprocess' thread can be used - but this last option
+  # seriously limits Galaxy's ability to scale. The default of 'auto'
+  # will use 'celery' if 'enable_celery_tasks' is set to true or
+  # 'inprocess' otherwise. This option serves as the default for all
+  # object stores and can be overridden on a per object store basis (but
+  # don't - just setup tmpwatch for all relevant cache paths).
+  #object_store_cache_monitor_driver: auto
+
+  # For object store cache monitoring done by Galaxy, this is the
+  # interval between cache checking steps. This is used by both
+  # inprocess cache monitors (which we recommend you do not use) and by
+  # the celery task if it is configured (by setting enable_celery_tasks
+  # to true and not setting object_store_cache_monitor_driver to
+  # external).
+  #object_store_cache_monitor_interval: 600
+
+  # Default cache path for caching object stores if cache not configured
+  # for that object store entry.
+  # The value of this option will be resolved with respect to
+  # <cache_dir>.
+  #object_store_cache_path: object_store_cache
+
+  # Default cache size for caching object stores if cache not configured
+  # for that object store entry.
+  #object_store_cache_size: -1
+
   # What Dataset attribute is used to reference files in an ObjectStore
   # implementation, this can be 'uuid' or 'id'. The default will depend
   # on how the object store is configured, starting with 20.05 Galaxy
diff --git a/lib/galaxy/config/sample/tool_shed.yml.sample b/lib/galaxy/config/sample/tool_shed.yml.sample
index 5f08dfbf430e..3b04d57f94a7 100644
--- a/lib/galaxy/config/sample/tool_shed.yml.sample
+++ b/lib/galaxy/config/sample/tool_shed.yml.sample
@@ -288,22 +288,15 @@ tool_shed:
   # store this information.
   #citation_cache_lock_dir: database/citations/lock
 
-  # When citation_cache_type = ext:database, this is
-  # the url of the database used by beaker for citation
-  # related caching.
-  # The application config code will set it to the 
-  # value of database_connection if this is not set.
+  # Url to database used by beaker for citation related caching.
   #citation_cache_url: null
 
-  # When citation_cache_type = ext:database, this is
-  # the database schema name of the table used by beaker
-  # for citation related caching.
-  #citation_cache_schema_name: null
+  # Database table used by beaker for citation related caching.
+  #citation_cache_table_name: beaker_cache
 
-  # When citation_cache_type = ext:database, this is
-  # the database table name used by beaker
-  # for citation related caching.
-  #citation_cache_table_name: null
+  # Schema of database table used by beaker for citation related
+  # caching.
+  #citation_cache_schema_name: null
 
   # Turn on logging of user actions to the database.  Actions currently
   # logged are grid views, tool searches, and use of "recently" used