diff --git a/conda-store-server/conda_store_server/_internal/server/views/api.py b/conda-store-server/conda_store_server/_internal/server/views/api.py index f22d8de00..6463a8498 100644 --- a/conda-store-server/conda_store_server/_internal/server/views/api.py +++ b/conda-store-server/conda_store_server/_internal/server/views/api.py @@ -1428,28 +1428,25 @@ async def api_get_settings( namespace: str = None, environment_name: str = None, ): - with conda_store.get_db() as db: - if namespace is None: - arn = "" - elif environment_name is None: - arn = namespace - else: - arn = f"{namespace}/{environment_name}" - - auth.authorize_request( - request, - arn, - {Permissions.SETTING_READ}, - require=True, - ) + if namespace is None: + arn = "" + elif environment_name is None: + arn = namespace + else: + arn = f"{namespace}/{environment_name}" + + auth.authorize_request( + request, + arn, + {Permissions.SETTING_READ}, + require=True, + ) - return { - "status": "ok", - "data": conda_store.get_settings( - namespace, environment_name - ).model_dump(), - "message": None, - } + return { + "status": "ok", + "data": conda_store.get_settings(namespace, environment_name).model_dump(), + "message": None, + } @router_api.put( @@ -1472,28 +1469,27 @@ async def api_put_settings( namespace: str = None, environment_name: str = None, ): - with conda_store.get_db() as db: - if namespace is None: - arn = "" - elif environment_name is None: - arn = namespace - else: - arn = f"{namespace}/{environment_name}" + if namespace is None: + arn = "" + elif environment_name is None: + arn = namespace + else: + arn = f"{namespace}/{environment_name}" + + auth.authorize_request( + request, + arn, + {Permissions.SETTING_UPDATE}, + require=True, + ) - auth.authorize_request( - request, - arn, - {Permissions.SETTING_UPDATE}, - require=True, - ) + try: + conda_store.set_settings(namespace, environment_name, data) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e.args[0])) - try: - conda_store.set_settings(namespace, environment_name, data) - except ValueError as e: - raise HTTPException(status_code=400, detail=str(e.args[0])) - - return { - "status": "ok", - "data": None, - "message": f"global setting keys {list(data.keys())} updated", - } + return { + "status": "ok", + "data": None, + "message": f"global setting keys {list(data.keys())} updated", + } diff --git a/conda-store-server/conda_store_server/_internal/server/views/ui.py b/conda-store-server/conda_store_server/_internal/server/views/ui.py index 30753176b..b91577a14 100644 --- a/conda-store-server/conda_store_server/_internal/server/views/ui.py +++ b/conda-store-server/conda_store_server/_internal/server/views/ui.py @@ -293,34 +293,33 @@ async def ui_get_setting( namespace: str = None, environment_name: str = None, ): - with conda_store.get_db() as db: - if namespace is None: - arn = "" - elif environment_name is None: - arn = namespace - else: - arn = f"{namespace}/{environment_name}" - - auth.authorize_request( - request, - arn, - {Permissions.SETTING_READ}, - require=True, - ) - - api_setting_url = str(request.url_for("api_put_settings")) - if namespace is not None: - api_setting_url += f"{namespace}/" - if environment_name is not None: - api_setting_url += f"{environment_name}/" - - context = { - "request": request, - "namespace": namespace, - "environment_name": environment_name, - "api_settings_url": api_setting_url, - "settings": conda_store.get_settings( - namespace=namespace, environment_name=environment_name - ), - } + if namespace is None: + arn = "" + elif environment_name is None: + arn = namespace + else: + arn = f"{namespace}/{environment_name}" + + auth.authorize_request( + request, + arn, + {Permissions.SETTING_READ}, + require=True, + ) + + api_setting_url = str(request.url_for("api_put_settings")) + if namespace is not None: + api_setting_url += f"{namespace}/" + if environment_name is not None: + api_setting_url += f"{environment_name}/" + + context = { + "request": request, + "namespace": namespace, + "environment_name": environment_name, + "api_settings_url": api_setting_url, + "settings": conda_store.get_settings( + namespace=namespace, environment_name=environment_name + ), + } return templates.TemplateResponse(request, "setting.html", context) diff --git a/conda-store-server/conda_store_server/_internal/settings.py b/conda-store-server/conda_store_server/_internal/settings.py index c4bd6f43a..6ac9b4c63 100644 --- a/conda-store-server/conda_store_server/_internal/settings.py +++ b/conda-store-server/conda_store_server/_internal/settings.py @@ -2,11 +2,11 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -from typing import Any, Dict, Callable import functools +from typing import Any, Callable, Dict -from sqlalchemy.orm import Session import pydantic +from sqlalchemy.orm import Session from conda_store_server import api from conda_store_server._internal import schema @@ -23,6 +23,7 @@ def wrapper(self, *args, **kwargs): result = func(self, *args, **kwargs) self.db.close() return result + return wrapper @_ensure_closed_session @@ -110,12 +111,18 @@ def get_settings( if len(prefixes) > 0: # get the fields that scoped globally. These are the keys that will NOT be # merged on for namespace and environment prefixes. - global_fields = [k for k, v in schema.Settings.model_fields.items() if v.json_schema_extra["metadata"]["global"]] + global_fields = [ + k + for k, v in schema.Settings.model_fields.items() + if v.json_schema_extra["metadata"]["global"] + ] # start building settings with the least specific defaults for prefix in prefixes: new_settings = api.get_kvstore_key_values(self.db, prefix) # remove any global fields - new_settings = {k: v for k, v in new_settings.items() if k not in global_fields} + new_settings = { + k: v for k, v in new_settings.items() if k not in global_fields + } settings.update(new_settings) return schema.Settings(**settings) @@ -123,7 +130,7 @@ def get_settings( @_ensure_closed_session def get_setting( self, key: str, namespace: str = None, environment_name: str = None - ) -> Any: + ): """Get a given setting at the given level of specificity. Will short cut and look up global setting directly even if a namespace/environment is specified @@ -142,11 +149,10 @@ def get_setting( Any setting value, merged for the given level of specificity """ - field = schema.Settings.model_fields.get(key) if field is None: - return - + return None + prefixes = ["setting"] if field.json_schema_extra["metadata"]["global"] is False: if namespace is not None: diff --git a/conda-store-server/conda_store_server/api.py b/conda-store-server/conda_store_server/api.py index 926c7b525..b21e45bdc 100644 --- a/conda-store-server/conda_store_server/api.py +++ b/conda-store-server/conda_store_server/api.py @@ -796,18 +796,18 @@ def get_kvstore_key_values(db, prefix: str): .all() } + def get_kvstore_key(db, prefix: str, key: str): """Get value for a particular prefix and key""" row = ( db.query(orm.KeyValueStore) - .filter(orm.KeyValueStore.prefix == prefix) - .filter(orm.KeyValueStore.key == key) - .first() + .filter(orm.KeyValueStore.prefix == prefix) + .filter(orm.KeyValueStore.key == key) + .first() ) if row is None: return None return row.value - def set_kvstore_key_values(db, prefix: str, d: Dict[str, Any], update: bool = True): diff --git a/conda-store-server/conda_store_server/conda_store.py b/conda-store-server/conda_store_server/conda_store.py index 129ed2a2a..c664faa79 100644 --- a/conda-store-server/conda_store_server/conda_store.py +++ b/conda-store-server/conda_store_server/conda_store.py @@ -8,14 +8,13 @@ from contextlib import contextmanager from typing import Any, Dict -import pydantic from celery import Celery, group from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.pool import QueuePool from conda_store_server import CONDA_STORE_DIR, api, conda_store_config, storage +from conda_store_server._internal import conda_utils, orm, schema, settings, utils from conda_store_server.exception import CondaStoreError -from conda_store_server._internal import conda_utils, orm, schema, utils, settings from conda_store_server.plugins import hookspec, plugin_manager from conda_store_server.plugins.types import lock @@ -72,8 +71,7 @@ def settings(self): # will release the connection, however, the connection may be restablished. # ref: https://docs.sqlalchemy.org/en/20/orm/session_basics.html#closing self._settings = settings.Settings( - db=db, - deployment_default=schema.Settings(**self.config.trait_values()) + db=db, deployment_default=schema.Settings(**self.config.trait_values()) ) return self._settings @@ -195,7 +193,7 @@ def get_settings( return self.settings.get_settings( namespace=namespace, environment_name=environment_name ) - + def get_setting( self, key: str, namespace: str = None, environment_name: str = None ) -> schema.Settings: diff --git a/conda-store-server/tests/_internal/test_settings.py b/conda-store-server/tests/_internal/test_settings.py index 5a1293ddc..62631a19c 100644 --- a/conda-store-server/tests/_internal/test_settings.py +++ b/conda-store-server/tests/_internal/test_settings.py @@ -14,69 +14,67 @@ @pytest.fixture def settings(db) -> Settings: default_settings = schema.Settings( - default_uid = 999, - default_gid = 999, - conda_channel_alias = "defaultchannelalias" + default_uid=999, default_gid=999, conda_channel_alias="defaultchannelalias" ) # setup test global settings global_settings = { "default_uid": 888, "conda_channel_alias": "globalchannelalias", - "conda_command": "myglobalcondacommand" + "conda_command": "myglobalcondacommand", } api.set_kvstore_key_values(db, "setting", global_settings) # setup test namespace settings - namespace_settings = { + namespace_settings = { "conda_channel_alias": "namespacechannelalias", "conda_command": "mynamespacecondacommand", - "conda_default_packages": ["ipykernel"] + "conda_default_packages": ["ipykernel"], } api.set_kvstore_key_values(db, "setting/test_namespace", namespace_settings) # setup test namespace (two) settings - namespace_two_settings = { + namespace_two_settings = { "conda_channel_alias": "namespacechannelalias", } api.set_kvstore_key_values(db, "setting/test_namespace_two", namespace_two_settings) # setup test environment settings - environment_settings = { + environment_settings = { "conda_channel_alias": "envchannelalias", - "conda_default_packages": ["numpy"] + "conda_default_packages": ["numpy"], } - api.set_kvstore_key_values(db, "setting/test_namespace/test_env", environment_settings) - - return Settings( - db=db, deployment_default=default_settings + api.set_kvstore_key_values( + db, "setting/test_namespace/test_env", environment_settings ) + return Settings(db=db, deployment_default=default_settings) + def test_ensure_session_is_closed(settings: Settings): # run a query against the db to start a transaction settings.get_settings() # ensure that the settings object cleans up it's transaction - assert settings.db.in_transaction() == False + assert not settings.db.in_transaction() @mock.patch("conda_store_server.api.get_kvstore_key_values") -def test_ensure_session_is_closed_on_error(mock_get_kvstore_key_values, settings: Settings): +def test_ensure_session_is_closed_on_error( + mock_get_kvstore_key_values, settings: Settings +): mock_get_kvstore_key_values.side_effect = Exception # run a query that will raise an exception - try: + with pytest.raises(Exception): settings.get_settings() - except: - pass - + # ensure that the settings object cleans up it's transaction - assert settings.db.in_transaction() == False - + assert not settings.db.in_transaction() + def test_get_settings_default(settings: Settings): test_settings = settings.get_settings() - + # ensure that we get the deployment default values assert test_settings.default_gid == 999 @@ -91,7 +89,7 @@ def test_get_settings_default(settings: Settings): def test_get_settings_namespace(settings: Settings): test_settings = settings.get_settings(namespace="test_namespace") - + # ensure that we get the deployment default values assert test_settings.default_gid == 999 @@ -112,7 +110,7 @@ def test_get_settings_namespace(settings: Settings): def test_get_settings_namespace_two(settings: Settings): test_settings = settings.get_settings(namespace="test_namespace_two") - + # ensure that we get the deployment default values assert test_settings.default_gid == 999 @@ -129,8 +127,10 @@ def test_get_settings_namespace_two(settings: Settings): def test_get_settings_environment(settings: Settings): - test_settings = settings.get_settings(namespace="test_namespace", environment_name="test_env") - + test_settings = settings.get_settings( + namespace="test_namespace", environment_name="test_env" + ) + # ensure that we get the deployment default values assert test_settings.default_gid == 999 @@ -150,7 +150,7 @@ def test_get_settings_namespace_dne(settings: Settings): # get settings for namespace that does not exist - we should # still get the default settings test_settings = settings.get_settings(namespace="idontexist") - + # ensure that we get the deployment default values assert test_settings.default_gid == 999 @@ -169,6 +169,7 @@ def test_set_settings_global_default(settings: Settings): check_settings = settings.get_settings(namespace="test_namespace") assert check_settings.default_uid == 0 + def test_set_settings_global_overriden_by_default(settings: Settings): # set test settings settings.set_settings(data={"conda_channel_alias": "newchanelalias"}) @@ -182,12 +183,14 @@ def test_set_settings_global_overriden_by_default(settings: Settings): def test_set_settings_invalid_setting_field(settings: Settings): with pytest.raises(ValueError, match=r"Invalid setting keys"): - settings.set_settings(data={"idontexist": "sure", "conda_channel_alias": "mynewalias"}) + settings.set_settings( + data={"idontexist": "sure", "conda_channel_alias": "mynewalias"} + ) def test_set_settings_invalid_setting_type(settings: Settings): with pytest.raises(ValueError, match=r"Invalid parsing of setting"): - settings.set_settings(data={"conda_channel_alias": [1,2,3]}) + settings.set_settings(data={"conda_channel_alias": [1, 2, 3]}) def test_set_settings_invalid_level(settings: Settings): @@ -206,7 +209,9 @@ def test_get_setting_invalid(settings: Settings): test_setting = settings.get_setting("notarealfield") assert test_setting is None - test_setting = settings.get_setting("notarealfield", namespace="test_namespace", environment_name="test_env") + test_setting = settings.get_setting( + "notarealfield", namespace="test_namespace", environment_name="test_env" + ) assert test_setting is None @@ -218,21 +223,29 @@ def test_get_setting_overriden(settings: Settings): test_setting = settings.get_setting("default_uid") assert test_setting == 888 # conda_command is also a global setting. Even if somehow the value gets - # injected into the db (as in this test setup), get_setting should honour + # injected into the db (as in this test setup), get_setting should honour # just the global setting test_setting = settings.get_setting("conda_command") assert test_setting == "myglobalcondacommand" - test_setting = settings.get_setting("conda_channel_alias", namespace="test_namespace") + test_setting = settings.get_setting( + "conda_channel_alias", namespace="test_namespace" + ) assert test_setting == "namespacechannelalias" test_setting = settings.get_setting("default_uid", namespace="test_namespace") assert test_setting == 888 test_setting = settings.get_setting("conda_command", namespace="test_namespace") assert test_setting == "myglobalcondacommand" - test_setting = settings.get_setting("conda_channel_alias", namespace="test_namespace", environment_name="test_env") + test_setting = settings.get_setting( + "conda_channel_alias", namespace="test_namespace", environment_name="test_env" + ) assert test_setting == "envchannelalias" - test_setting = settings.get_setting("default_uid", namespace="test_namespace", environment_name="test_env") + test_setting = settings.get_setting( + "default_uid", namespace="test_namespace", environment_name="test_env" + ) assert test_setting == 888 - test_setting = settings.get_setting("conda_command", namespace="test_namespace", environment_name="test_env") + test_setting = settings.get_setting( + "conda_command", namespace="test_namespace", environment_name="test_env" + ) assert test_setting == "myglobalcondacommand" diff --git a/conda-store-server/tests/test_api.py b/conda-store-server/tests/test_api.py index 4454eab2c..608958a7b 100644 --- a/conda-store-server/tests/test_api.py +++ b/conda-store-server/tests/test_api.py @@ -345,25 +345,26 @@ def test_get_set_keyvaluestore(db): assert setting_3 == api.get_kvstore_key_values(db, "pytest/1/2") # check get_kvstore_value - assert 2 == api.get_kvstore_key(db, "pytest", "b") - assert 2 == api.get_kvstore_key(db, "pytest/1", "d") - assert 2 == api.get_kvstore_key(db, "pytest/1/2", "f") + assert api.get_kvstore_key(db, "pytest", "b") == 2 + assert api.get_kvstore_key(db, "pytest/1", "d") == 2 + assert api.get_kvstore_key(db, "pytest/1/2", "f") == 2 # test updating a prefix api.set_kvstore_key_values(db, "pytest", setting_2) assert api.get_kvstore_key_values(db, "pytest") == {**setting_1, **setting_2} - assert 2 == api.get_kvstore_key(db, "pytest", "d") + assert api.get_kvstore_key(db, "pytest", "d") == 2 # test updating a prefix api.set_kvstore_key_values(db, "pytest", {"c": 999, "d": 999}, update=False) assert api.get_kvstore_key_values(db, "pytest") == {**setting_1, **setting_2} - assert 2 == api.get_kvstore_key(db, "pytest", "d") + assert api.get_kvstore_key(db, "pytest", "d") == 2 def test_get_kvstore_key_dne(db): # db starts empty, try to get a value that does not exist assert api.get_kvstore_key(db, "pytest", "c") is None + def test_build_path_too_long(db, conda_store, simple_specification): conda_store.config.store_directory = "A" * 800 build_id = conda_store.register_environment( diff --git a/docusaurus-docs/conda-store/references/configuring-conda-store.md b/docusaurus-docs/conda-store/references/configuring-conda-store.md index 64f5ca734..c9f1674c1 100644 --- a/docusaurus-docs/conda-store/references/configuring-conda-store.md +++ b/docusaurus-docs/conda-store/references/configuring-conda-store.md @@ -4,7 +4,7 @@ description: configuring conda-store # Configuring conda-store -The conda-store server has two types of config. +The conda-store server has two types of config. * server configuration config, eg. url for database endpoints, redis endpoints, port to run on, etc. @@ -16,29 +16,29 @@ The server configuration is always specified in the [conda-store configuration f There are two options to set the application settings config. These are through: -* the conda-store configuration file. See the available options in the +* the conda-store configuration file. See the available options in the [configuration options doc](./configuration-options.md) * the conda-store admin console. Available at `/admin` -There are multiple levels which settings config can be applied to conda store. +There are multiple levels which settings config can be applied to conda store. * deployment defaults (controlled by the config file) * global defaults (controlled by the admin console) * namespace defaults (controlled by the admin console) * environment default (controlled by the admin console) -The most specific settings config will take precedence over the more -general setting. For example, if the `conda_command` is specified in +The most specific settings config will take precedence over the more +general setting. For example, if the `conda_command` is specified in all four settings config levels, the most specific, environment settings config will be applied. :::note Since the deployment defaults are of lowest precedent when settings are being generated, it is likely that they will be overridden by another -by another level, for example the global defaults. +by another level, for example the global defaults. So, it is recommended that users change global defaults from the admin console. ::: -The full list of application settings config is described in the +The full list of application settings config is described in the [settings pydantic model](https://github.com/conda-incubator/conda-store/blob/main/conda-store-server/conda_store_server/_internal/schema.py#L203).