Merge pull request #50 from lonvia/more-cmdline-options

Make location of config file configurable on the command line

MANIFEST.in

@@ -1,4 +1,4 @@
 include src/nominatim_data_analyser/rules_specifications/*.yaml
-include src/nominatim_data_analyser/config/default.yaml
+include src/nominatim_data_analyser/default_config.yaml
 include src/nominatim_data_analyser/py.typed
 recursive-include contrib *

README.md

@@ -30,7 +30,11 @@ the database DSN by supplying a custom config.yaml file.
 
 ## Configuration file
 
-To modify the configuration, you need to copy the ```analyser/config/default.yaml``` to ```analyser/config/config.yaml``` and you can modify the values inside the config.yaml file.
+Some parts of the analyser are configurable. You can find the default
+configuration in `src/nominatim_data_analyser/default_config.yaml`. To
+modify the configuration create a copy of the file with the modified
+values and put it as `config.yaml` in the directory where the analyser
+is executed.
 
 ## Frontend set up
 
@@ -45,6 +49,7 @@ Analysis is run with the nominatim-data-analyser tool:
 * --execute-all: Executes all QA rules.
 * --filter [rules_names…]: Filters some QA rules so they are not executed.
 * --execute-one <rule_name>: Executes the given QA rule.
+* --config: Set a custom location for the configuration file.
 
 During development you can run the same tool directly from the source tree
 after having built everything using the supplied `cli.py`.

pyproject.toml

@@ -34,7 +34,6 @@ tests = [
 
 [tool.setuptools]
 packages = ["nominatim_data_analyser",
-            "nominatim_data_analyser.config",
             "nominatim_data_analyser.logger",
             "nominatim_data_analyser.core",
             "nominatim_data_analyser.core.exceptions",

src/nominatim_data_analyser/cli.py

@@ -1,6 +1,7 @@
-from .core.core import Core
 import argparse
 
+from .core.core import Core
+
 def cli() -> int:
     parser = argparse.ArgumentParser(prog='nominatim-analyser')
 
@@ -10,17 +11,20 @@ def cli() -> int:
                         help='Filters some QA rules so they are not executed.')
     parser.add_argument('--execute-one', metavar='<QA rule name>', action='store',
                         type=str, help='Executes the given QA rule')
+    parser.add_argument('--config', metavar='<YAML file>', default='config.yaml',
+                        help='Location of config file (default: config.yaml)')
 
     args = parser.parse_args()
 
+    core = Core(config_file=args.config)
+
     # Executes all the QA rules. If a filter is given, these rules are excluded from the execution.
     if args.execute_all:
-        if args.filter:
-            Core().execute_all(args.filter)
-        else:
-            Core().execute_all()
+        core.execute_all(args.filter)
     elif args.execute_one:
         # Execute the given QA rule.
-        Core().execute_one(args.execute_one)
+        core.execute_one(args.execute_one)
+    else:
+        return 1
 
     return 0

src/nominatim_data_analyser/config.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+
+import yaml
+
+from .logger.logger import LOG
+
+
+class Config:
+    values: dict[str, str] = {}
+
+
+def load_config(config_file: Path | None) -> None:
+    """
+       Load the YAML config file into the
+       config global variable.
+    """
+    Config.values.clear()
+
+    # First load the default settings.
+    _get_config_file_contents(Path(__file__, '..', 'default_config.yaml').resolve())
+
+    # Then overwrite with potential custom settings.
+    if config_file is not None and config_file.is_file():
+        LOG.info(f"Loading config from {config_file}.")
+        _get_config_file_contents(config_file)
+
+
+def _get_config_file_contents(config_file: Path) -> None:
+    with config_file.open('r') as file:
+        try:
+            contents = yaml.safe_load(file)
+        except yaml.YAMLError as exc:
+            LOG.error(f"Error while loading the config file: {exc}")
+            raise
+
+        if not isinstance(contents, dict):
+            raise RuntimeError('Error in config file, expected key-value entries.')
+
+        for k, v in contents.items():
+            if not isinstance(k, str):
+                raise RuntimeError(f"Error in config file, non-string key {k}.")
+            Config.values[k] = str(v)

src/nominatim_data_analyser/core/core.py

@@ -2,15 +2,15 @@
 from .assembler.pipeline_assembler import PipelineAssembler
 from ..logger.logger import LOG
 from ..logger.timer import Timer
-from ..config import Config
+from ..config import load_config
 from pathlib import Path
 
 class Core():
     """
         Core of the analyser used to execute rules.
     """
-    def __init__(self) -> None:
-        Config.load_config()
+    def __init__(self, config_file: str | Path | None) -> None:
+        load_config(None if config_file is None else Path(config_file))
         self.rules_path = Path(__file__, '..', '..', 'rules_specifications').resolve()
 
     def execute_all(self, filter: list[str] | None = None) -> None:
@@ -22,13 +22,16 @@ def execute_all(self, filter: list[str] | None = None) -> None:
         """
         for rule_file in self.rules_path.glob('*.yaml'):
             if not filter or rule_file.stem not in filter:
-                self.execute_one(rule_file.stem)
+                self._execute(rule_file)
 
     def execute_one(self, name: str) -> None:
         """
             Execute one QA rule based on its YAML file name.
         """
+        self._execute(self.rules_path / f"{name}.yaml")
+
+    def _execute(self, rule_file: Path) -> None:
         timer = Timer().start_timer()
-        loaded_yaml = load_yaml_rule(name)
-        PipelineAssembler(loaded_yaml, name).assemble().process_and_next()
-        LOG.info('<%s> The whole rule executed in %s mins %s secs', name, *timer.get_elapsed())
+        loaded_yaml = load_yaml_rule(rule_file)
+        PipelineAssembler(loaded_yaml, rule_file.stem).assemble().process_and_next()
+        LOG.info('<%s> The whole rule executed in %s mins %s secs', rule_file.stem, *timer.get_elapsed())

src/nominatim_data_analyser/core/pipes/data_fetching/sql_processor.py

@@ -2,7 +2,7 @@
 
 import psycopg
 
-from ....config.config import Config
+from ....config import Config
 from ... import Pipe
 from ....logger.timer import Timer
 

src/nominatim_data_analyser/core/yaml_logic/yaml_loader.py

@@ -1,35 +1,34 @@
 from typing import Any, cast
+from pathlib import Path
+
+import yaml
+
 from ..dynamic_value.switch import Switch
 from ..dynamic_value.variable import Variable
 from ..assembler import PipelineAssembler
 from .. import Pipe
 from ...logger.logger import LOG
-from pathlib import Path
-import yaml
-
-base_rules_path = Path(__file__, '..', '..', '..', 'rules_specifications').resolve()
 
-def load_yaml_rule(file_name: str) -> dict[str, Any]:
+def load_yaml_rule(rule_file: Path) -> dict[str, Any]:
     """
         Load the YAML specification file.
         YAML constructors are added to handle custom types in the YAML.
     """
     def _sub_pipeline(loader: yaml.SafeLoader, node: yaml.Node) -> Pipe:
         if not isinstance(node, yaml.MappingNode):
             raise RuntimeError("!switch expects mapping.")
-        return sub_pipeline_constructor(loader, node, file_name)
+        return sub_pipeline_constructor(loader, node, rule_file.stem)
 
     yaml.add_constructor(u'!sub-pipeline', _sub_pipeline, Loader=yaml.SafeLoader)
     yaml.add_constructor(u'!variable', variable_constructor, Loader=yaml.SafeLoader)
     yaml.add_constructor(u'!switch', switch_constructor, Loader=yaml.SafeLoader)
 
-    path = Path(base_rules_path / Path(file_name + '.yaml')).resolve()
-    with open(str(path), 'r') as file:
+    with rule_file.open('r') as file:
         try:
             loaded = cast(dict[str, Any], yaml.safe_load(file))
         except yaml.YAMLError as exc:
             LOG.error('Error while loading the YAML rule file %s: %s',
-                      file_name, exc)
+                      rule_file.stem, exc)
             raise
 
     return loaded

src/nominatim_data_analyser/default_config.yaml

@@ -0,0 +1,9 @@
+# Data source name for the database connection.
+Dsn: 'dbname=nominatim'
+
+# Path to the folder where rules data are stored (geojson, vector tiles, etc).
+RulesFolderPath: 'qa-data'
+
+# Prefix path of the web URL to access the rules data (ex: https://nominatim.org/QA-data).
+WebPrefixPath: ''
+

tests/config/test_config.py

@@ -1,31 +1,39 @@
-
-from nominatim_data_analyser.config import Config
-from pathlib import Path
 import pytest
 import yaml
 
-def test_load_default_config(config: Config) -> None:
+from nominatim_data_analyser.config import Config, load_config
+
+def test_load_default_config() -> None:
     """
         Test the load_config() method. The default config should be
         returned because no config.yaml file is present in the
         default_config folder used as the config_folder_path.
     """
-    config.load_config(Path(__file__).parent / 'default_config')
-    assert config.values == {'Dsn': 'default_dsn'}
+    load_config(None)
+    assert Config.values['Dsn'] == 'dbname=nominatim'
+    assert Config.values['RulesFolderPath'] == 'qa-data'
 
-def test_load_custom_config(config: Config) -> None:
+def test_load_custom_config(tmp_path) -> None:
     """
         Test the load_config() method. The custom config should be
         returned because one config.yaml file is present in the
         custom_config folder used as the config_folder_path.
     """
-    config.load_config(Path(__file__).parent / 'custom_config')
-    assert config.values == {'Dsn': 'custom_dsn'}
+    cfgfile = tmp_path / 'myconfig.yaml'
+    cfgfile.write_text("Dsn: 'custom_dsn'")
+
+    load_config(cfgfile)
 
-def test_load_broken_config(config: Config) -> None:
+    assert Config.values['Dsn'] == 'custom_dsn'
+    assert Config.values['RulesFolderPath'] == 'qa-data'
+
+def test_load_broken_config(tmp_path) -> None:
     """
         Test the load_config() method. A YAMLError exception should
         be raised as the config file has a wrong syntax.
     """
+    cfgfile = tmp_path / 'myconfig.yaml'
+    cfgfile.write_text(">>>>>>>>Dsn: 'custom_dsn'")
+
     with pytest.raises(yaml.YAMLError):
-        config.load_config(Path(__file__).parent / 'broken_config')
+        load_config(cfgfile)

tests/conftest.py

@@ -17,7 +17,7 @@
     sys.path.insert(0, str(SRC_DIR / BUILD_DIR))
 
 
-from nominatim_data_analyser.config import Config
+from nominatim_data_analyser.config import Config, load_config
 from nominatim_data_analyser.core.pipes import FillingPipe
 from nominatim_data_analyser.core.pipes.data_fetching.sql_processor import SQLProcessor
 from nominatim_data_analyser.core.pipes.data_processing import (GeometryConverter,
@@ -73,7 +73,7 @@ def config() -> Config:
     """
         Loads the config and returns it.
     """
-    Config.load_config()
+    load_config(None)
     return Config
 
 @pytest.fixture