From 6a363cfdcd67b5d2b24428961edc594f8308d781 Mon Sep 17 00:00:00 2001 From: Nok Date: Tue, 14 Nov 2023 07:45:33 +0000 Subject: [PATCH 01/14] update release notes Signed-off-by: Nok --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index 9734e3f449..55aa331181 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -8,6 +8,7 @@ * Added the functionality to choose a merging strategy for config files loaded with `OmegaConfigLoader`. * Modified the mechanism of importing datasets, raise more explicit error when dependencies are missing. * Added validation for configuration file used to override run commands via the CLI. +* Moved the default environment `base` and `local` from config loader to `_ProjectSettings`. This enable the use of config loader as a standalone class without affecting existing Kedro Project users. ## Bug fixes and other changes * Added a new field `add-ons` to `pyproject.toml` when a project is created. From 80b2a789dd81074b348269741e6b68436bacada7 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 15 Nov 2023 15:27:17 +0800 Subject: [PATCH 02/14] Remove the default env in config loader - add the env in tests Signed-off-by: Nok Chan --- kedro/config/omegaconf_config.py | 8 +- tests/config/test_omegaconf_config.py | 147 ++++++++++++++++++++------ 2 files changed, 118 insertions(+), 37 deletions(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index c75a7713c9..f8ec549b38 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -80,8 +80,8 @@ def __init__( # noqa: PLR0913 runtime_params: dict[str, Any] = None, *, config_patterns: dict[str, list[str]] = None, - base_env: str = "base", - default_run_env: str = "local", + base_env: str = None, + default_run_env: str = None, custom_resolvers: dict[str, Callable] = None, merge_strategy: dict[str, str] = None, ): @@ -102,8 +102,8 @@ def __init__( # noqa: PLR0913 custom_resolvers: A dictionary of custom resolvers to be registered. For more information, see here: https://omegaconf.readthedocs.io/en/2.3_branch/custom_resolvers.html#custom-resolvers """ - self.base_env = base_env - self.default_run_env = default_run_env + self.base_env = base_env or "" + self.default_run_env = default_run_env or "" self.merge_strategy = merge_strategy or {} self.config_patterns = { diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index 31d8be44f0..aaf8762f15 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -158,7 +158,9 @@ class TestOmegaConfigLoader: @use_config_dir def test_load_core_config_dict_syntax(self, tmp_path): """Make sure core config can be fetched with a dict [] access.""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) params = conf["parameters"] catalog = conf["catalog"] @@ -168,7 +170,9 @@ def test_load_core_config_dict_syntax(self, tmp_path): @use_config_dir def test_load_core_config_get_syntax(self, tmp_path): """Make sure core config can be fetched with .get()""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) params = conf.get("parameters") catalog = conf.get("catalog") @@ -179,7 +183,9 @@ def test_load_core_config_get_syntax(self, tmp_path): def test_load_local_config_overrides_base(self, tmp_path): """Make sure that configs from `local/` override the ones from `base/`""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) params = conf["parameters"] catalog = conf["catalog"] @@ -193,14 +199,18 @@ def test_load_local_config_overrides_base(self, tmp_path): def test_load_base_config(self, tmp_path, base_config): """Test config loading if `local/` directory is empty""" (tmp_path / _DEFAULT_RUN_ENV).mkdir(exist_ok=True) - catalog = OmegaConfigLoader(str(tmp_path))["catalog"] + catalog = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["catalog"] assert catalog == base_config @use_proj_catalog def test_duplicate_patterns(self, tmp_path, base_config): """Test config loading if the glob patterns cover the same file""" (tmp_path / _DEFAULT_RUN_ENV).mkdir(exist_ok=True) - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) catalog1 = conf["catalog"] catalog2 = conf["catalog"] assert catalog1 == catalog2 == base_config @@ -212,16 +222,24 @@ def test_subdirs_dont_exist(self, tmp_path, base_config): r"or is not a valid directory\: {}" ) with pytest.raises(MissingConfigException, match=pattern.format(".*base")): - OmegaConfigLoader(str(tmp_path))["catalog"] + OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["catalog"] with pytest.raises(MissingConfigException, match=pattern.format(".*local")): proj_catalog = tmp_path / _BASE_ENV / "catalog.yml" _write_yaml(proj_catalog, base_config) - OmegaConfigLoader(str(tmp_path))["catalog"] + print( + OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["catalog"] + ) @pytest.mark.usefixtures("create_config_dir", "proj_catalog", "proj_catalog_nested") def test_nested(self, tmp_path): """Test loading the config from subdirectories""" - config_loader = OmegaConfigLoader(str(tmp_path)) + config_loader = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) config_loader.default_run_env = "prod" prod_catalog = tmp_path / "prod" / "catalog.yml" @@ -246,7 +264,9 @@ def test_nested_subdirs_duplicate(self, tmp_path, base_config): r"\: cars, trains" ) with pytest.raises(ValueError, match=pattern): - OmegaConfigLoader(str(tmp_path))["catalog"] + OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["catalog"] @use_config_dir def test_multiple_nested_subdirs_duplicates( @@ -277,7 +297,9 @@ def test_multiple_nested_subdirs_duplicates( ) with pytest.raises(ValueError) as exc: - OmegaConfigLoader(str(tmp_path))["catalog"] + OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["catalog"] assert re.search(pattern_catalog_nested, str(exc.value)) assert re.search(pattern_catalog_local, str(exc.value)) assert re.search(pattern_nested_local, str(exc.value)) @@ -290,14 +312,18 @@ def test_bad_config_syntax(self, tmp_path): pattern = f"Invalid YAML or JSON file {conf_path.as_posix()}" with pytest.raises(ParserError, match=re.escape(pattern)): - OmegaConfigLoader(str(tmp_path))["catalog"] + OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["catalog"] def test_lots_of_duplicates(self, tmp_path): data = {str(i): i for i in range(100)} _write_yaml(tmp_path / _BASE_ENV / "catalog1.yml", data) _write_yaml(tmp_path / _BASE_ENV / "catalog2.yml", data) - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) pattern = ( r"Duplicate keys found in " r"(.*catalog2\.yml and .*catalog1\.yml|.*catalog1\.yml and .*catalog2\.yml)" @@ -335,7 +361,12 @@ def test_cannot_load_non_yaml_or_json_files(self, tmp_path): db_config_path = tmp_path / _BASE_ENV / "db.ini" _write_dummy_ini(db_config_path) - conf = OmegaConfigLoader(str(tmp_path), config_patterns=db_patterns) + conf = OmegaConfigLoader( + str(tmp_path), + config_patterns=db_patterns, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, + ) pattern = ( r"No files of YAML or JSON format found in " r".*base or " @@ -357,7 +388,9 @@ def test_no_files_found(self, tmp_path): r"\[\'credentials\*\', \'credentials\*/\**\', \'\**/credentials\*\'\]" ) with pytest.raises(MissingConfigException, match=pattern): - OmegaConfigLoader(str(tmp_path))["credentials"] + OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["credentials"] def test_empty_catalog_file(self, tmp_path): """Check that empty catalog file is read and returns an empty dict""" @@ -420,7 +453,9 @@ def test_yaml_parser_error(self, tmp_path): f" line 3, position 10." ) with pytest.raises(ParserError, match=re.escape(msg)): - OmegaConfigLoader(str(tmp_path))["catalog"] + OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["catalog"] def test_customised_config_patterns(self, tmp_path): config_loader = OmegaConfigLoader( @@ -444,9 +479,12 @@ def test_customised_config_patterns(self, tmp_path): def test_default_destructive_merging_strategy(self, tmp_path, mlflow_config): mlflow_patterns = {"mlflow": ["mlflow*", "mlflow*/**", "**/mlflow*"]} - conf = OmegaConfigLoader(str(tmp_path), config_patterns=mlflow_patterns)[ - "mlflow" - ] + conf = OmegaConfigLoader( + str(tmp_path), + config_patterns=mlflow_patterns, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, + )["mlflow"] assert conf == { "tracking": { @@ -462,6 +500,8 @@ def test_destructive_merging_strategy(self, tmp_path, mlflow_config): str(tmp_path), config_patterns=mlflow_patterns, merge_strategy={"mlflow": "destructive"}, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, )["mlflow"] assert conf == { @@ -478,6 +518,8 @@ def test_soft_merging_strategy(self, tmp_path, mlflow_config): str(tmp_path), config_patterns=mlflow_patterns, merge_strategy={"mlflow": "soft"}, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, )["mlflow"] assert conf == { @@ -497,12 +539,16 @@ def test_unsupported_merge_strategy(self, tmp_path, mlflow_config): str(tmp_path), config_patterns=mlflow_patterns, merge_strategy={"mlflow": "hard"}, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, )["mlflow"] @use_config_dir def test_adding_extra_keys_to_confloader(self, tmp_path): """Make sure extra keys can be added directly to the config loader instance.""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) catalog = conf["catalog"] conf["spark"] = {"spark_config": "emr.blabla"} @@ -513,7 +559,9 @@ def test_adding_extra_keys_to_confloader(self, tmp_path): def test_bypass_catalog_config_loading(self, tmp_path): """Make sure core config loading can be bypassed by setting the key and values directly on the config loader instance.""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) conf["catalog"] = {"catalog_config": "something_new"} assert conf["catalog"] == {"catalog_config": "something_new"} @@ -522,7 +570,9 @@ def test_bypass_catalog_config_loading(self, tmp_path): @use_credentials_env_variable_yml def test_load_credentials_from_env_variables(self, tmp_path): """Load credentials from environment variables""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) os.environ["TEST_USERNAME"] = "test_user" os.environ["TEST_KEY"] = "test_key" assert conf["credentials"]["user"]["name"] == "test_user" @@ -532,7 +582,9 @@ def test_load_credentials_from_env_variables(self, tmp_path): @use_catalog_env_variable_yml def test_env_resolver_not_used_for_catalog(self, tmp_path): """Check that the oc.env resolver is not used for catalog loading""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) os.environ["TEST_DATASET"] = "test_dataset" with pytest.raises(errors.UnsupportedInterpolationType): conf["catalog"]["test"]["file_path"] @@ -542,7 +594,9 @@ def test_env_resolver_not_used_for_catalog(self, tmp_path): def test_env_resolver_is_cleared_after_loading(self, tmp_path): """Check that the ``oc.env`` resolver is cleared after loading credentials in the case that it was not registered beforehand.""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) os.environ["TEST_USERNAME"] = "test_user" os.environ["TEST_KEY"] = "test_key" assert conf["credentials"]["user"]["name"] == "test_user" @@ -553,7 +607,9 @@ def test_env_resolver_is_cleared_after_loading(self, tmp_path): def test_env_resolver_is_registered_after_loading(self, tmp_path): """Check that the ``oc.env`` resolver is registered after loading credentials in the case that it was registered beforehand""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) OmegaConf.register_new_resolver("oc.env", oc.env) os.environ["TEST_USERNAME"] = "test_user" os.environ["TEST_KEY"] = "test_key" @@ -574,7 +630,11 @@ def test_load_config_from_tar_file(self, tmp_path): ] ) - conf = OmegaConfigLoader(conf_source=f"{tmp_path}/tar_conf.tar.gz") + conf = OmegaConfigLoader( + conf_source=f"{tmp_path}/tar_conf.tar.gz", + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, + ) catalog = conf["catalog"] assert catalog["trains"]["type"] == "MemoryDataset" @@ -598,14 +658,20 @@ def zipdir(path, ziph): ) as zipf: zipdir(tmp_path, zipf) - conf = OmegaConfigLoader(conf_source=f"{tmp_path}/Python.zip") + conf = OmegaConfigLoader( + conf_source=f"{tmp_path}/Python.zip", + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, + ) catalog = conf["catalog"] assert catalog["trains"]["type"] == "MemoryDataset" @use_config_dir def test_variable_interpolation_with_correct_env(self, tmp_path): """Make sure the parameters is interpolated with the correct environment""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) params = conf["parameters"] # Making sure it is not override by local/parameters_global.yml assert params["interpolated_param"] == "base" @@ -629,6 +695,8 @@ def test_runtime_params_not_propogate_non_parameters_config(self, tmp_path): str(tmp_path), config_patterns={"spark": ["spark*", "spark*/**", "**/spark*"]}, runtime_params=runtime_params, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, ) parameters = conf["parameters"] catalog = conf["catalog"] @@ -646,7 +714,9 @@ def test_ignore_hidden_keys(self, tmp_path): _write_yaml(tmp_path / _BASE_ENV / "catalog1.yml", {"k1": "v1", "_k2": "v2"}) _write_yaml(tmp_path / _BASE_ENV / "catalog2.yml", {"k3": "v3", "_k2": "v4"}) - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) conf.default_run_env = "" catalog = conf["catalog"] assert catalog.keys() == {"k1", "k3"} @@ -671,7 +741,9 @@ def test_variable_interpolation_in_catalog_with_templates(self, tmp_path): } _write_yaml(base_catalog, catalog_config) - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) conf.default_run_env = "" assert conf["catalog"]["companies"]["type"] == "pandas.CSVDataset" @@ -690,7 +762,9 @@ def test_variable_interpolation_in_catalog_with_separate_templates_file( _write_yaml(base_catalog, catalog_config) _write_yaml(tmp_catalog, template) - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) conf.default_run_env = "" assert conf["catalog"]["companies"]["type"] == "pandas.CSVDataset" @@ -710,7 +784,12 @@ def test_custom_resolvers(self, tmp_path): "oc.env": oc.env, } os.environ["VAR"] = "my_env_variable" - conf = OmegaConfigLoader(tmp_path, custom_resolvers=custom_resolvers) + conf = OmegaConfigLoader( + tmp_path, + custom_resolvers=custom_resolvers, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, + ) conf.default_run_env = "" assert conf["parameters"]["model_options"]["param1"] == 7 assert conf["parameters"]["model_options"]["param2"] == 3 @@ -722,7 +801,7 @@ def test_globals(self, tmp_path): "x": 0, } _write_yaml(globals_params, globals_config) - conf = OmegaConfigLoader(tmp_path, default_run_env="") + conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) # OmegaConfigLoader has globals resolver assert OmegaConf.has_resolver("globals") # Globals is readable in a dict way @@ -887,7 +966,9 @@ def test_bad_globals_underscore(self, tmp_path): "hidden_path", ["/User/.hidden/dummy.yml", "/User/dummy/.hidden.yml"] ) def test_is_hidden_config(self, tmp_path, hidden_path): - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) assert conf._is_hidden(hidden_path) @pytest.mark.parametrize( From ab67d37b45f9ffcf6a6745e82d05acebf513fafb Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 15 Nov 2023 15:52:27 +0800 Subject: [PATCH 03/14] remove unncessary `default_run_env` Signed-off-by: Nok Chan --- tests/config/test_omegaconf_config.py | 50 ++++++++++++++++----------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index aaf8762f15..af79ee3ef8 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -422,7 +422,10 @@ def test_overlapping_patterns(self, tmp_path, mocker): } catalog = OmegaConfigLoader( - conf_source=str(tmp_path), env="dev", config_patterns=catalog_patterns + conf_source=str(tmp_path), + base_env=_BASE_ENV, + env="dev", + config_patterns=catalog_patterns, )["catalog"] expected_catalog = { "env": "dev", @@ -679,7 +682,12 @@ def test_variable_interpolation_with_correct_env(self, tmp_path): @use_config_dir def test_runtime_params_override_interpolated_value(self, tmp_path): """Make sure interpolated value is updated correctly with runtime_params""" - conf = OmegaConfigLoader(str(tmp_path), runtime_params={"test_env": "dummy"}) + conf = OmegaConfigLoader( + str(tmp_path), + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, + runtime_params={"test_env": "dummy"}, + ) params = conf["parameters"] assert params["interpolated_param"] == "dummy" @@ -741,10 +749,7 @@ def test_variable_interpolation_in_catalog_with_templates(self, tmp_path): } _write_yaml(base_catalog, catalog_config) - conf = OmegaConfigLoader( - str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV - ) - conf.default_run_env = "" + conf = OmegaConfigLoader(str(tmp_path), base_env=_BASE_ENV) assert conf["catalog"]["companies"]["type"] == "pandas.CSVDataset" def test_variable_interpolation_in_catalog_with_separate_templates_file( @@ -762,10 +767,7 @@ def test_variable_interpolation_in_catalog_with_separate_templates_file( _write_yaml(base_catalog, catalog_config) _write_yaml(tmp_catalog, template) - conf = OmegaConfigLoader( - str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV - ) - conf.default_run_env = "" + conf = OmegaConfigLoader(str(tmp_path), base_env=_BASE_ENV) assert conf["catalog"]["companies"]["type"] == "pandas.CSVDataset" def test_custom_resolvers(self, tmp_path): @@ -788,9 +790,7 @@ def test_custom_resolvers(self, tmp_path): tmp_path, custom_resolvers=custom_resolvers, base_env=_BASE_ENV, - default_run_env=_DEFAULT_RUN_ENV, ) - conf.default_run_env = "" assert conf["parameters"]["model_options"]["param1"] == 7 assert conf["parameters"]["model_options"]["param2"] == 3 assert conf["parameters"]["model_options"]["param3"] == "my_env_variable" @@ -825,7 +825,7 @@ def test_globals_resolution(self, tmp_path): _write_yaml(base_params, param_config) _write_yaml(globals_params, globals_config) _write_yaml(base_catalog, catalog_config) - conf = OmegaConfigLoader(tmp_path, default_run_env="") + conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) # Globals are resolved correctly in parameter assert conf["parameters"]["my_param"] == globals_config["x"] # The default value is used if the key does not exist @@ -848,7 +848,7 @@ def test_globals_nested(self, tmp_path): } _write_yaml(base_params, param_config) _write_yaml(globals_params, globals_config) - conf = OmegaConfigLoader(tmp_path, default_run_env="") + conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) assert conf["parameters"]["my_param"] == globals_config["x"] # Nested globals are accessible with dot notation assert conf["parameters"]["my_nested_param"] == globals_config["nested"]["y"] @@ -875,7 +875,9 @@ def test_globals_across_env(self, tmp_path): _write_yaml(local_params, local_param_config) _write_yaml(base_globals, base_globals_config) _write_yaml(local_globals, local_globals_config) - conf = OmegaConfigLoader(tmp_path) + conf = OmegaConfigLoader( + tmp_path, base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) # Local global overwrites the base global value assert conf["parameters"]["param1"] == local_globals_config["y"] # Base global value is accessible to local params @@ -892,7 +894,7 @@ def test_globals_default(self, tmp_path): base_globals_config = {"x": {"DUMMY": 3}} _write_yaml(base_params, base_param_config) _write_yaml(base_globals, base_globals_config) - conf = OmegaConfigLoader(tmp_path, default_run_env="") + conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) # Default value is being used as int assert conf["parameters"]["int"] == 1 # Default value is being used as str @@ -916,7 +918,7 @@ def test_globals_default_none(self, tmp_path): } _write_yaml(base_params, base_param_config) _write_yaml(base_globals, base_globals_config) - conf = OmegaConfigLoader(tmp_path, default_run_env="") + conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) # Default value can be 0 or null assert conf["parameters"]["zero"] == 0 assert conf["parameters"]["null"] is None @@ -936,7 +938,7 @@ def test_globals_missing_default(self, tmp_path): } _write_yaml(base_params, param_config) _write_yaml(globals_params, globals_config) - conf = OmegaConfigLoader(tmp_path, default_run_env="") + conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) with pytest.raises( InterpolationResolutionError, @@ -955,7 +957,7 @@ def test_bad_globals_underscore(self, tmp_path): } _write_yaml(base_params, base_param_config) _write_yaml(base_globals, base_globals_config) - conf = OmegaConfigLoader(tmp_path, default_run_env="") + conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) with pytest.raises( InterpolationResolutionError, match=r"Keys starting with '_' are not supported for globals.", @@ -1030,7 +1032,10 @@ def test_runtime_params_resolution(self, tmp_path): _write_yaml(base_params, param_config) _write_yaml(base_catalog, catalog_config) conf = OmegaConfigLoader( - tmp_path, default_run_env="", runtime_params=runtime_params + tmp_path, + base_env=_BASE_ENV, + default_run_env="", + runtime_params=runtime_params, ) # runtime are resolved correctly in parameter assert conf["parameters"]["my_runtime_param"] == runtime_params["x"] @@ -1049,7 +1054,10 @@ def test_runtime_params_missing_default(self, tmp_path): } _write_yaml(base_params, param_config) conf = OmegaConfigLoader( - tmp_path, default_run_env="", runtime_params=runtime_params + tmp_path, + base_env=_BASE_ENV, + default_run_env="", + runtime_params=runtime_params, ) with pytest.raises( InterpolationResolutionError, From 6ecbdef7a6e1efd0a7afe303ece20d4982b15088 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 15 Nov 2023 15:55:45 +0800 Subject: [PATCH 04/14] fix tests Signed-off-by: Nok Chan --- tests/config/test_omegaconf_config.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index af79ee3ef8..805a7409a9 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -1086,15 +1086,10 @@ def test_runtime_params_in_globals_not_allowed(self, tmp_path): ): OmegaConfigLoader( tmp_path, - base_env="", - default_run_env="local", + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, runtime_params=runtime_params, ) - with pytest.raises( - UnsupportedInterpolationType, - match=r"The `runtime_params:` resolver is not supported for globals.", - ): - OmegaConfigLoader(tmp_path, runtime_params=runtime_params) def test_runtime_params_default_global(self, tmp_path): base_globals = tmp_path / _BASE_ENV / "globals.yml" @@ -1116,7 +1111,7 @@ def test_runtime_params_default_global(self, tmp_path): _write_yaml(base_catalog, catalog_config) _write_yaml(base_globals, globals_config) conf = OmegaConfigLoader( - tmp_path, default_run_env="", runtime_params=runtime_params + tmp_path, base_env=_BASE_ENV, runtime_params=runtime_params ) # runtime params are resolved correctly in catalog using global default assert conf["catalog"]["companies"]["type"] == globals_config["dataset"]["type"] @@ -1164,7 +1159,12 @@ def test_unsupported_interpolation_globals(self, tmp_path): UnsupportedInterpolationType, match=r"Unsupported interpolation type non_existent_resolver", ): - OmegaConfigLoader(tmp_path, runtime_params=runtime_params) + OmegaConfigLoader( + tmp_path, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, + runtime_params=runtime_params, + ) with pytest.raises( UnsupportedInterpolationType, match=r"Unsupported interpolation type non_existent_resolver", From 7731883614b50da11a063b4c2a4a22e72518d17b Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 15 Nov 2023 16:10:19 +0800 Subject: [PATCH 05/14] Added a couple of tests to make sure config loader standalone mode works Signed-off-by: Nok Chan --- tests/config/test_omegaconf_config.py | 225 ++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index 805a7409a9..cd86d9eb6c 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -1196,3 +1196,228 @@ def test_override_globals(self, tmp_path): } assert conf["parameters"]["my_global"] == 89 assert conf["parameters"]["my_second_global"] == 24 + +class TestOmegaConfigLoaderStandalone: + """ + Test OmegaConfigLoader in standalone mode (defaults settings without environments) + """ + def test_load_config_without_local(self, tmp_path): + base_catalog = tmp_path / "catalog.yml" + catalog_config = { + "companies": { + "type": "MemoryDataset", + "filepath": "data/01_raw/companies.csv", + }, + } + _write_yaml(base_catalog, catalog_config) + + conf = OmegaConfigLoader( + tmp_path + ) + assert conf["catalog"]["companies"]["type"] == catalog_config["companies"]["type"] + + + def test_variable_interpolation_in_catalog_with_templates(self, tmp_path): + base_catalog = tmp_path / "catalog.yml" + catalog_config = { + "companies": { + "type": "${_pandas.type}", + "filepath": "data/01_raw/companies.csv", + }, + "_pandas": {"type": "pandas.CSVDataset"}, + } + _write_yaml(base_catalog, catalog_config) + + conf = OmegaConfigLoader(str(tmp_path), ) + assert conf["catalog"]["companies"]["type"] == "pandas.CSVDataset" + def test_variable_interpolation_in_catalog_with_separate_templates_file( + self, tmp_path + ): + base_catalog = tmp_path / "catalog.yml" + catalog_config = { + "companies": { + "type": "${_pandas.type}", + "filepath": "data/01_raw/companies.csv", + } + } + tmp_catalog = tmp_path / "catalog_temp.yml" + template = {"_pandas": {"type": "pandas.CSVDataset"}} + _write_yaml(base_catalog, catalog_config) + _write_yaml(tmp_catalog, template) + + conf = OmegaConfigLoader(str(tmp_path)) + assert conf["catalog"]["companies"]["type"] == "pandas.CSVDataset" + + def test_globals(self, tmp_path): + globals_params = tmp_path / "globals.yml" + globals_config = { + "x": 0, + } + _write_yaml(globals_params, globals_config) + conf = OmegaConfigLoader(tmp_path) + # OmegaConfigLoader has globals resolver + assert OmegaConf.has_resolver("globals") + # Globals is readable in a dict way + assert conf["globals"] == globals_config + def test_globals_resolution(self, tmp_path): + base_params = tmp_path / "parameters.yml" + base_catalog = tmp_path / "catalog.yml" + globals_params = tmp_path / "globals.yml" + param_config = { + "my_param": "${globals:x}", + "my_param_default": "${globals:y,34}", # y does not exist in globals + } + catalog_config = { + "companies": { + "type": "${globals:dataset_type}", + "filepath": "data/01_raw/companies.csv", + }, + } + globals_config = {"x": 34, "dataset_type": "pandas.CSVDataset"} + _write_yaml(base_params, param_config) + _write_yaml(globals_params, globals_config) + _write_yaml(base_catalog, catalog_config) + conf = OmegaConfigLoader(tmp_path) + # Globals are resolved correctly in parameter + assert conf["parameters"]["my_param"] == globals_config["x"] + # The default value is used if the key does not exist + assert conf["parameters"]["my_param_default"] == 34 + # Globals are resolved correctly in catalog + assert conf["catalog"]["companies"]["type"] == globals_config["dataset_type"] + + def test_globals_nested(self, tmp_path): + base_params = tmp_path / "parameters.yml" + globals_params = tmp_path / "globals.yml" + param_config = { + "my_param": "${globals:x}", + "my_nested_param": "${globals:nested.y}", + } + globals_config = { + "x": 34, + "nested": { + "y": 42, + }, + } + _write_yaml(base_params, param_config) + _write_yaml(globals_params, globals_config) + conf = OmegaConfigLoader(tmp_path) + assert conf["parameters"]["my_param"] == globals_config["x"] + # Nested globals are accessible with dot notation + assert conf["parameters"]["my_nested_param"] == globals_config["nested"]["y"] + + def test_globals_default(self, tmp_path): + base_params = tmp_path / "parameters.yml" + base_globals = tmp_path / "globals.yml" + base_param_config = { + "int": "${globals:x.NOT_EXIST, 1}", + "str": "${globals: x.NOT_EXIST, '2'}", + "dummy": "${globals: x.DUMMY.DUMMY, '2'}", + } + base_globals_config = {"x": {"DUMMY": 3}} + _write_yaml(base_params, base_param_config) + _write_yaml(base_globals, base_globals_config) + conf = OmegaConfigLoader(tmp_path) + # Default value is being used as int + assert conf["parameters"]["int"] == 1 + # Default value is being used as str + assert conf["parameters"]["str"] == "2" + # Test when x.DUMMY is not a dictionary it should still work + assert conf["parameters"]["dummy"] == "2" + def test_globals_default_none(self, tmp_path): + base_params = tmp_path / "parameters.yml" + base_globals = tmp_path / "globals.yml" + base_param_config = { + "zero": "${globals: x.NOT_EXIST, 0}", + "null": "${globals: x.NOT_EXIST, null}", + "null2": "${globals: x.y}", + } + base_globals_config = { + "x": { + "z": 23, + "y": None, + }, + } + _write_yaml(base_params, base_param_config) + _write_yaml(base_globals, base_globals_config) + conf = OmegaConfigLoader(tmp_path) + # Default value can be 0 or null + assert conf["parameters"]["zero"] == 0 + assert conf["parameters"]["null"] is None + # Global value is null + assert conf["parameters"]["null2"] is None + + def test_runtime_params_resolution(self, tmp_path): + base_params = tmp_path / "parameters.yml" + base_catalog = tmp_path / "catalog.yml" + runtime_params = { + "x": 45, + "dataset": { + "type": "pandas.CSVDataset", + }, + } + param_config = { + "my_runtime_param": "${runtime_params:x}", + "my_param_default": "${runtime_params:y,34}", # y does not exist in globals + } + catalog_config = { + "companies": { + "type": "${runtime_params:dataset.type}", + "filepath": "data/01_raw/companies.csv", + }, + } + _write_yaml(base_params, param_config) + _write_yaml(base_catalog, catalog_config) + conf = OmegaConfigLoader( + tmp_path, + runtime_params=runtime_params, + ) + # runtime are resolved correctly in parameter + assert conf["parameters"]["my_runtime_param"] == runtime_params["x"] + # The default value is used if the key does not exist + assert conf["parameters"]["my_param_default"] == 34 + # runtime params are resolved correctly in catalog + assert conf["catalog"]["companies"]["type"] == runtime_params["dataset"]["type"] + + def test_runtime_params_in_globals_not_allowed(self, tmp_path): + base_globals = tmp_path / "globals.yml" + runtime_params = { + "x": 45, + } + base_globals_config = { + "my_global_var": "${runtime_params:x}", + } + + _write_yaml(base_globals, base_globals_config) + + with pytest.raises( + UnsupportedInterpolationType, + match=r"The `runtime_params:` resolver is not supported for globals.", + ): + OmegaConfigLoader( + tmp_path, + runtime_params=runtime_params, + ) + def test_runtime_params_default_global(self, tmp_path): + base_globals = tmp_path / "globals.yml" + base_catalog = tmp_path / "catalog.yml" + runtime_params = { + "x": 45, + } + globals_config = { + "dataset": { + "type": "pandas.CSVDataset", + } + } + catalog_config = { + "companies": { + "type": "${runtime_params:type, ${globals:dataset.type, 'MemoryDataset'}}", + "filepath": "data/01_raw/companies.csv", + }, + } + _write_yaml(base_catalog, catalog_config) + _write_yaml(base_globals, globals_config) + conf = OmegaConfigLoader( + tmp_path, runtime_params=runtime_params + ) + # runtime params are resolved correctly in catalog using global default + assert conf["catalog"]["companies"]["type"] == globals_config["dataset"]["type"] \ No newline at end of file From 417874be5857c2aecb47df22064182e78c93143a Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 15 Nov 2023 17:27:49 +0800 Subject: [PATCH 06/14] fix tests Signed-off-by: Nok Chan --- tests/config/test_omegaconf_config.py | 49 ++++++++++++++----------- tests/framework/context/test_context.py | 21 ++++++++--- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index cd86d9eb6c..1a3e862e9e 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -1197,10 +1197,12 @@ def test_override_globals(self, tmp_path): assert conf["parameters"]["my_global"] == 89 assert conf["parameters"]["my_second_global"] == 24 + class TestOmegaConfigLoaderStandalone: """ Test OmegaConfigLoader in standalone mode (defaults settings without environments) """ + def test_load_config_without_local(self, tmp_path): base_catalog = tmp_path / "catalog.yml" catalog_config = { @@ -1211,11 +1213,10 @@ def test_load_config_without_local(self, tmp_path): } _write_yaml(base_catalog, catalog_config) - conf = OmegaConfigLoader( - tmp_path + conf = OmegaConfigLoader(tmp_path) + assert ( + conf["catalog"]["companies"]["type"] == catalog_config["companies"]["type"] ) - assert conf["catalog"]["companies"]["type"] == catalog_config["companies"]["type"] - def test_variable_interpolation_in_catalog_with_templates(self, tmp_path): base_catalog = tmp_path / "catalog.yml" @@ -1228,19 +1229,22 @@ def test_variable_interpolation_in_catalog_with_templates(self, tmp_path): } _write_yaml(base_catalog, catalog_config) - conf = OmegaConfigLoader(str(tmp_path), ) + conf = OmegaConfigLoader( + str(tmp_path), + ) assert conf["catalog"]["companies"]["type"] == "pandas.CSVDataset" + def test_variable_interpolation_in_catalog_with_separate_templates_file( self, tmp_path ): - base_catalog = tmp_path / "catalog.yml" + base_catalog = tmp_path / "catalog.yml" catalog_config = { "companies": { "type": "${_pandas.type}", "filepath": "data/01_raw/companies.csv", } } - tmp_catalog = tmp_path / "catalog_temp.yml" + tmp_catalog = tmp_path / "catalog_temp.yml" template = {"_pandas": {"type": "pandas.CSVDataset"}} _write_yaml(base_catalog, catalog_config) _write_yaml(tmp_catalog, template) @@ -1249,7 +1253,7 @@ def test_variable_interpolation_in_catalog_with_separate_templates_file( assert conf["catalog"]["companies"]["type"] == "pandas.CSVDataset" def test_globals(self, tmp_path): - globals_params = tmp_path / "globals.yml" + globals_params = tmp_path / "globals.yml" globals_config = { "x": 0, } @@ -1259,9 +1263,10 @@ def test_globals(self, tmp_path): assert OmegaConf.has_resolver("globals") # Globals is readable in a dict way assert conf["globals"] == globals_config + def test_globals_resolution(self, tmp_path): - base_params = tmp_path / "parameters.yml" - base_catalog = tmp_path / "catalog.yml" + base_params = tmp_path / "parameters.yml" + base_catalog = tmp_path / "catalog.yml" globals_params = tmp_path / "globals.yml" param_config = { "my_param": "${globals:x}", @@ -1286,8 +1291,8 @@ def test_globals_resolution(self, tmp_path): assert conf["catalog"]["companies"]["type"] == globals_config["dataset_type"] def test_globals_nested(self, tmp_path): - base_params = tmp_path / "parameters.yml" - globals_params = tmp_path / "globals.yml" + base_params = tmp_path / "parameters.yml" + globals_params = tmp_path / "globals.yml" param_config = { "my_param": "${globals:x}", "my_nested_param": "${globals:nested.y}", @@ -1306,8 +1311,8 @@ def test_globals_nested(self, tmp_path): assert conf["parameters"]["my_nested_param"] == globals_config["nested"]["y"] def test_globals_default(self, tmp_path): - base_params = tmp_path / "parameters.yml" - base_globals = tmp_path / "globals.yml" + base_params = tmp_path / "parameters.yml" + base_globals = tmp_path / "globals.yml" base_param_config = { "int": "${globals:x.NOT_EXIST, 1}", "str": "${globals: x.NOT_EXIST, '2'}", @@ -1323,8 +1328,9 @@ def test_globals_default(self, tmp_path): assert conf["parameters"]["str"] == "2" # Test when x.DUMMY is not a dictionary it should still work assert conf["parameters"]["dummy"] == "2" + def test_globals_default_none(self, tmp_path): - base_params = tmp_path / "parameters.yml" + base_params = tmp_path / "parameters.yml" base_globals = tmp_path / "globals.yml" base_param_config = { "zero": "${globals: x.NOT_EXIST, 0}", @@ -1347,8 +1353,8 @@ def test_globals_default_none(self, tmp_path): assert conf["parameters"]["null2"] is None def test_runtime_params_resolution(self, tmp_path): - base_params = tmp_path / "parameters.yml" - base_catalog = tmp_path / "catalog.yml" + base_params = tmp_path / "parameters.yml" + base_catalog = tmp_path / "catalog.yml" runtime_params = { "x": 45, "dataset": { @@ -1379,7 +1385,7 @@ def test_runtime_params_resolution(self, tmp_path): assert conf["catalog"]["companies"]["type"] == runtime_params["dataset"]["type"] def test_runtime_params_in_globals_not_allowed(self, tmp_path): - base_globals = tmp_path / "globals.yml" + base_globals = tmp_path / "globals.yml" runtime_params = { "x": 45, } @@ -1397,6 +1403,7 @@ def test_runtime_params_in_globals_not_allowed(self, tmp_path): tmp_path, runtime_params=runtime_params, ) + def test_runtime_params_default_global(self, tmp_path): base_globals = tmp_path / "globals.yml" base_catalog = tmp_path / "catalog.yml" @@ -1416,8 +1423,6 @@ def test_runtime_params_default_global(self, tmp_path): } _write_yaml(base_catalog, catalog_config) _write_yaml(base_globals, globals_config) - conf = OmegaConfigLoader( - tmp_path, runtime_params=runtime_params - ) + conf = OmegaConfigLoader(tmp_path, runtime_params=runtime_params) # runtime params are resolved correctly in catalog using global default - assert conf["catalog"]["companies"]["type"] == globals_config["dataset"]["type"] \ No newline at end of file + assert conf["catalog"]["companies"]["type"] == globals_config["dataset"]["type"] diff --git a/tests/framework/context/test_context.py b/tests/framework/context/test_context.py index f5dc49e81b..f152c27358 100644 --- a/tests/framework/context/test_context.py +++ b/tests/framework/context/test_context.py @@ -16,7 +16,7 @@ from pandas.testing import assert_frame_equal from kedro import __version__ as kedro_version -from kedro.config import MissingConfigException, OmegaConfigLoader +from kedro.config import MissingConfigException from kedro.framework.context import KedroContext from kedro.framework.context.context import ( _convert_paths_to_absolute_posix, @@ -27,9 +27,10 @@ from kedro.framework.project import ( ValidationError, _ProjectSettings, - configure_project, pipelines, ) +from kedro.framework.session import KedroSession +from kedro.framework.startup import bootstrap_project MOCK_PACKAGE_NAME = "mock_package_name" @@ -123,14 +124,19 @@ def prepare_project_dir(tmp_path, base_config, local_config, env): parameters = tmp_path / "conf" / "base" / "parameters.json" db_config_path = tmp_path / "conf" / "base" / "db.ini" project_parameters = {"param1": 1, "param2": 2, "param3": {"param4": 3}} + + # Create configurations _write_yaml(proj_catalog, base_config) _write_yaml(env_catalog, local_config) _write_yaml(env_credentials, local_config) _write_json(parameters, project_parameters) - _write_dummy_ini(db_config_path) _write_toml(tmp_path / "pyproject.toml", pyproject_toml_payload) + # Create the necessary files in src/ + (tmp_path / "src" / MOCK_PACKAGE_NAME).mkdir(parents=True, exist_ok=True) + (tmp_path / "src" / MOCK_PACKAGE_NAME / "__init__.py").write_text(" ") + @pytest.fixture def mock_settings_file_bad_data_catalog_class(tmpdir): @@ -189,8 +195,13 @@ def extra_params(request): @pytest.fixture def dummy_context(tmp_path, prepare_project_dir, env, extra_params): - configure_project(MOCK_PACKAGE_NAME) - config_loader = OmegaConfigLoader(str(tmp_path / "conf"), env=env) + bootstrap_project(tmp_path) + + session = KedroSession.create( + project_path=tmp_path, env=env, extra_params=extra_params + ) + context = session.load_context() + config_loader = context.config_loader context = KedroContext( MOCK_PACKAGE_NAME, str(tmp_path), From f295f3a6260a92395cff7682b7a660e1b030f835 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 15 Nov 2023 17:31:41 +0800 Subject: [PATCH 07/14] lint Signed-off-by: Nok Chan --- kedro/framework/project/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kedro/framework/project/__init__.py b/kedro/framework/project/__init__.py index 485df57891..85eafc37c9 100644 --- a/kedro/framework/project/__init__.py +++ b/kedro/framework/project/__init__.py @@ -105,7 +105,9 @@ class _ProjectSettings(LazySettings): "CONFIG_LOADER_CLASS", default=_get_default_class("kedro.config.OmegaConfigLoader"), ) - _CONFIG_LOADER_ARGS = Validator("CONFIG_LOADER_ARGS", default={}) + _CONFIG_LOADER_ARGS = Validator( + "CONFIG_LOADER_ARGS", default={"base_env": "base", "default_run_env": "local"} + ) _DATA_CATALOG_CLASS = _IsSubclassValidator( "DATA_CATALOG_CLASS", default=_get_default_class("kedro.io.DataCatalog") ) From 6ddf591bf6f694e4cf19f85da2bda8381a2e3599 Mon Sep 17 00:00:00 2001 From: Nok Date: Thu, 16 Nov 2023 06:02:39 +0000 Subject: [PATCH 08/14] Fix tests Signed-off-by: Nok --- tests/framework/project/test_settings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/framework/project/test_settings.py b/tests/framework/project/test_settings.py index d74fb60d5b..74a2ac50ca 100644 --- a/tests/framework/project/test_settings.py +++ b/tests/framework/project/test_settings.py @@ -90,7 +90,10 @@ def test_settings_without_configure_project_shows_default_values(): assert settings.CONTEXT_CLASS is KedroContext assert settings.CONF_SOURCE == "conf" assert settings.CONFIG_LOADER_CLASS == OmegaConfigLoader - assert settings.CONFIG_LOADER_ARGS == {} + assert settings.CONFIG_LOADER_ARGS == { + "base_env": "base", + "default_run_env": "local", + } assert settings.DATA_CATALOG_CLASS == DataCatalog From 0a574d6b93e34eeed9bf3006d933e85d9f799e34 Mon Sep 17 00:00:00 2001 From: Nok Date: Thu, 16 Nov 2023 06:02:50 +0000 Subject: [PATCH 09/14] update default Signed-off-by: Nok --- .../src/{{ cookiecutter.python_package }}/settings.py | 2 ++ .../src/{{ cookiecutter.python_package }}/settings.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py index 86a92b1c80..3207b1da0a 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py @@ -26,6 +26,8 @@ # CONFIG_LOADER_CLASS = OmegaConfigLoader # Keyword arguments to pass to the `CONFIG_LOADER_CLASS` constructor. # CONFIG_LOADER_ARGS = { +# "base_env": "base", +# "default_run_env": "local", # "config_patterns": { # "spark" : ["spark*/"], # "parameters": ["parameters*", "parameters*/**", "**/parameters*"], diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py b/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py index 53c6f763bc..db58cab52d 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py @@ -29,6 +29,8 @@ # Keyword arguments to pass to the `CONFIG_LOADER_CLASS` constructor. # CONFIG_LOADER_ARGS = { +# "base_env": "base", +# "default_run_env": "local", # "config_patterns": { # "spark" : ["spark*/"], # "parameters": ["parameters*", "parameters*/**", "**/parameters*"], From 42ba4f90fb0d30a0f65e7d860993fcac484032b3 Mon Sep 17 00:00:00 2001 From: Nok Date: Thu, 16 Nov 2023 06:36:50 +0000 Subject: [PATCH 10/14] fix test coverage Signed-off-by: Nok --- tests/config/test_omegaconf_config.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index 1a3e862e9e..0e42a2ff13 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -1071,13 +1071,10 @@ def test_runtime_params_in_globals_not_allowed(self, tmp_path): runtime_params = { "x": 45, } - base_globals_config = { - "my_global_var": "${runtime_params:x}", - } local_globals_config = { "my_local_var": "${runtime_params:x}", # x does exist but shouldn't be allowed in globals } - _write_yaml(base_globals, base_globals_config) + _write_yaml(local_globals, local_globals_config) with pytest.raises( From 585a472d43f7a0d391c36f4d897b843823a964ed Mon Sep 17 00:00:00 2001 From: Nok Date: Thu, 16 Nov 2023 07:06:01 +0000 Subject: [PATCH 11/14] fix test and lint Signed-off-by: Nok --- tests/config/test_omegaconf_config.py | 7 +++---- tests/framework/context/test_context.py | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index 0e42a2ff13..00788a0c9e 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -1067,15 +1067,14 @@ def test_runtime_params_missing_default(self, tmp_path): def test_runtime_params_in_globals_not_allowed(self, tmp_path): base_globals = tmp_path / _BASE_ENV / "globals.yml" - local_globals = tmp_path / _DEFAULT_RUN_ENV / "globals.yml" runtime_params = { "x": 45, } - local_globals_config = { - "my_local_var": "${runtime_params:x}", # x does exist but shouldn't be allowed in globals + base_globals_config = { + "my_global_var": "${runtime_params:x}", # x does exist but shouldn't be allowed in globals } - _write_yaml(local_globals, local_globals_config) + _write_yaml(base_globals, base_globals_config) with pytest.raises( UnsupportedInterpolationType, diff --git a/tests/framework/context/test_context.py b/tests/framework/context/test_context.py index f152c27358..ac4a046c61 100644 --- a/tests/framework/context/test_context.py +++ b/tests/framework/context/test_context.py @@ -122,7 +122,6 @@ def prepare_project_dir(tmp_path, base_config, local_config, env): env_catalog = tmp_path / "conf" / str(env) / "catalog.yml" env_credentials = tmp_path / "conf" / str(env) / "credentials.yml" parameters = tmp_path / "conf" / "base" / "parameters.json" - db_config_path = tmp_path / "conf" / "base" / "db.ini" project_parameters = {"param1": 1, "param2": 2, "param3": {"param4": 3}} # Create configurations From b7d46c3662193399e4fa65b00e7fdae59b8c2c9a Mon Sep 17 00:00:00 2001 From: Nok Date: Mon, 20 Nov 2023 07:27:06 +0000 Subject: [PATCH 12/14] rename test to 'without_environment' to reflect the change of defaults Signed-off-by: Nok --- tests/config/test_omegaconf_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index 00788a0c9e..e7ed511b2d 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -1199,7 +1199,7 @@ class TestOmegaConfigLoaderStandalone: Test OmegaConfigLoader in standalone mode (defaults settings without environments) """ - def test_load_config_without_local(self, tmp_path): + def test_load_config_without_environment(self, tmp_path): base_catalog = tmp_path / "catalog.yml" catalog_config = { "companies": { From 1c30daa3a2d807587dee9080689968424c256270 Mon Sep 17 00:00:00 2001 From: Nok Date: Mon, 20 Nov 2023 08:19:24 +0000 Subject: [PATCH 13/14] fix tests according to comments Signed-off-by: Nok --- RELEASE.md | 3 +- tests/config/test_omegaconf_config.py | 40 ++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 6492ef3343..2bf0b1c5b4 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -8,8 +8,7 @@ * Added the functionality to choose a merging strategy for config files loaded with `OmegaConfigLoader`. * Modified the mechanism of importing datasets, raise more explicit error when dependencies are missing. * Added validation for configuration file used to override run commands via the CLI. -* Moved the default environment `base` and `local` from config loader to `_ProjectSettings`. This enable the use of config loader as a standalone class without affecting existing Kedro Project users. - +* Moved the default environment `base` and `local` from config loader to `_ProjectSettings`. This enables the use of config loader as a standalone class without affecting existing Kedro Framework users. ## Bug fixes and other changes * Added a new field `add-ons` to `pyproject.toml` when a project is created. * Reduced `spaceflights` data to minimise waiting times during tutorial execution. diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index e7ed511b2d..da005e2e99 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -722,9 +722,7 @@ def test_ignore_hidden_keys(self, tmp_path): _write_yaml(tmp_path / _BASE_ENV / "catalog1.yml", {"k1": "v1", "_k2": "v2"}) _write_yaml(tmp_path / _BASE_ENV / "catalog2.yml", {"k3": "v3", "_k2": "v4"}) - conf = OmegaConfigLoader( - str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV - ) + conf = OmegaConfigLoader(str(tmp_path), base_env=_BASE_ENV) conf.default_run_env = "" catalog = conf["catalog"] assert catalog.keys() == {"k1", "k3"} @@ -1067,15 +1065,31 @@ def test_runtime_params_missing_default(self, tmp_path): def test_runtime_params_in_globals_not_allowed(self, tmp_path): base_globals = tmp_path / _BASE_ENV / "globals.yml" + local_globals = tmp_path / _DEFAULT_RUN_ENV / "globals.yml" + runtime_params = { "x": 45, } + base_globals_config = { - "my_global_var": "${runtime_params:x}", # x does exist but shouldn't be allowed in globals + "my_global_var": "${runtime_params:x}", + } + local_globals_config = { + "my_local_var": "${runtime_params:x}", # x does exist but shouldn't be allowed in globals } _write_yaml(base_globals, base_globals_config) + _write_yaml(local_globals, local_globals_config) + with pytest.raises( + UnsupportedInterpolationType, + match=r"The `runtime_params:` resolver is not supported for globals.", + ): + OmegaConfigLoader( + tmp_path, + default_run_env=_DEFAULT_RUN_ENV, + runtime_params=runtime_params, + ) with pytest.raises( UnsupportedInterpolationType, match=r"The `runtime_params:` resolver is not supported for globals.", @@ -1214,6 +1228,24 @@ def test_load_config_without_environment(self, tmp_path): conf["catalog"]["companies"]["type"] == catalog_config["companies"]["type"] ) + def test_load_config_only_base_environment(self, tmp_path): + base_parameters = tmp_path / _BASE_ENV / "parameters.yml" + base_parameters_config = {"dummy": "base"} + + _write_yaml(base_parameters, base_parameters_config) + + conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) + assert conf["parameters"]["dummy"] == "base" + + def test_load_config_only_default_run_environment(self, tmp_path): + default_env_parameters = tmp_path / _DEFAULT_RUN_ENV / "parameters.yml" + default_env_parameters_config = {"dummy": "default"} + + _write_yaml(default_env_parameters, default_env_parameters_config) + + conf = OmegaConfigLoader(tmp_path, default_run_env=_DEFAULT_RUN_ENV) + assert conf["parameters"]["dummy"] == "default" + def test_variable_interpolation_in_catalog_with_templates(self, tmp_path): base_catalog = tmp_path / "catalog.yml" catalog_config = { From aa02d6c931a52ed5d5c5f01ee908a62e059a90db Mon Sep 17 00:00:00 2001 From: Nok Date: Tue, 21 Nov 2023 05:37:21 +0000 Subject: [PATCH 14/14] update test to test arbitary env explicitly Signed-off-by: Nok --- tests/config/test_omegaconf_config.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index da005e2e99..640def26d4 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -723,7 +723,6 @@ def test_ignore_hidden_keys(self, tmp_path): _write_yaml(tmp_path / _BASE_ENV / "catalog2.yml", {"k3": "v3", "_k2": "v4"}) conf = OmegaConfigLoader(str(tmp_path), base_env=_BASE_ENV) - conf.default_run_env = "" catalog = conf["catalog"] assert catalog.keys() == {"k1", "k3"} @@ -1229,21 +1228,23 @@ def test_load_config_without_environment(self, tmp_path): ) def test_load_config_only_base_environment(self, tmp_path): - base_parameters = tmp_path / _BASE_ENV / "parameters.yml" + dummy_env = "dummy_base_env" + base_parameters = tmp_path / dummy_env / "parameters.yml" base_parameters_config = {"dummy": "base"} _write_yaml(base_parameters, base_parameters_config) - conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) + conf = OmegaConfigLoader(tmp_path, base_env=dummy_env) assert conf["parameters"]["dummy"] == "base" def test_load_config_only_default_run_environment(self, tmp_path): - default_env_parameters = tmp_path / _DEFAULT_RUN_ENV / "parameters.yml" + dummy_env = "dummy_base_env" + default_env_parameters = tmp_path / dummy_env / "parameters.yml" default_env_parameters_config = {"dummy": "default"} _write_yaml(default_env_parameters, default_env_parameters_config) - conf = OmegaConfigLoader(tmp_path, default_run_env=_DEFAULT_RUN_ENV) + conf = OmegaConfigLoader(tmp_path, default_run_env=dummy_env) assert conf["parameters"]["dummy"] == "default" def test_variable_interpolation_in_catalog_with_templates(self, tmp_path):