CMIP7 DataRequest Integration Tests

.github/workflows/CI-test.yaml

@@ -44,6 +44,7 @@ jobs:
           export XARRAY_ENGINE=h5netcdf
           export PREFECT_SERVER_EPHEMERAL_STARTUP_TIMEOUT_SECONDS=300
           pytest -vvv -s --cov tests/unit/*.py
+          pytest -vvv -s --cov tests/unit/data_request/*.py
       - name: Test with pytest (Integration)
         run: |
           export XARRAY_ENGINE=h5netcdf
@@ -80,18 +81,18 @@ jobs:
           python -m pip install .[dev]
       - name: Run isort
         run: |
-          isort --profile black --check --skip ./cmip6-cmor-tables --skip ./versioneer.py .
+          isort --profile black --check --skip ./cmip6-cmor-tables --skip ./versioneer.py --skip ./CMIP7_DReq_Software .
       - name: Lint with flake8
         run: |
           ## stop the build if there are Python syntax errors or undefined names
           #flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           ## exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           #flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
           # stop at any error
-          flake8 . --show-source --statistics --exclude ./cmip6-cmor-tables,./build,_version.py,./src/pymorize/webapp.py
+          flake8 . --show-source --statistics --exclude ./cmip6-cmor-tables,./build,_version.py,./src/pymorize/webapp.py,./CMIP7_DReq_Software
       - name: Run black
         run: |
-          black --check --extend-exclude 'cmip6-cmor-tables|versioneer\.py|webapp\.py' .
+          black --check --extend-exclude 'cmip6-cmor-tables|CMIP7_DReq_Software|versioneer\.py|webapp\.py' .
       - name: yamllint
         run: |
-          yamllint -d "{extends: default, rules: {line-length: {level: 'warning'}}, ignore: ['cmip6-cmor-tables']}" .
+          yamllint -d "{extends: default, rules: {line-length: {level: 'warning'}}, ignore: ['cmip6-cmor-tables', 'CMIP7_DReq_Software']}" .

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "cmip6-cmor-tables"]
 	path = cmip6-cmor-tables
 	url = https://github.com/PCMDI/cmip6-cmor-tables.git
+[submodule "CMIP7_DReq_Software"]
+	path = CMIP7_DReq_Software
+	url = https://github.com/CMIP-Data-Request/CMIP7_DReq_Software.git

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
 default_language_version:
   python: python3.9
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -22,16 +22,17 @@
     rev: '7.1.1'
     hooks:
       - id: flake8
         # YAML stuff from https://earth.bsc.es/gitlab/digital-twins/de_340-2/workflow
   - repo: https://github.com/adrienverge/yamllint.git
     rev: v1.35.1 # or higher tag
     hooks:
       - id: yamllint
         args: [-d, "{extends: default, rules: {line-length: {level: 'warning'}}}"]
 exclude: |
   (?x)  # Enable verbose regex
   (
       ^versioneer\.py$|           # Exclude 'versioneer.py'
       ^src/pymorize/webapp\.py$|  # Exclude 'src/pymorize/webapp.py'
-      _version\.py                # Exclude '_version.py'
+      _version\.py|               # Exclude '_version.py'
+      ^src/pymorize/data/cmip7/   # Exclude 'src/pymorize/data/cmip7/'
   )

conftest.py

@@ -1,9 +1,3 @@
-import os
-import re
-from pathlib import Path
-
-import pytest
-
 from tests.utils.constants import TEST_ROOT  # noqa: F401
 
 pytest_plugins = [
@@ -21,29 +15,3 @@
     "tests.fixtures.CV_Dir",
     "tests.fixtures.data_requests",
 ]
-
-
-@pytest.hookimpl(tryfirst=True)
-def pytest_collection_modifyitems(config, items):
-    for item in items:
-        if item.fspath and item.fspath.ext == ".py":
-            item.add_marker(pytest.mark.doctest)
-
-
-@pytest.fixture(autouse=True)
-def pathlib_doctest_directive(doctest_namespace):
-    """Replace PosixPath/WindowsPath with Path in doc-test output."""
-    doctest_namespace["Path"] = Path
-
-    def path_replace(output):
-        """Replace platform-specific Path output with generic Path in doc-tests."""
-        return re.sub(r"(PosixPath|WindowsPath)\((.*?)\)", r"Path(\2)", output)
-
-    doctest_namespace["path_replace"] = path_replace
-
-
-def pytest_unconfigure(config):
-    """Remove all JSON files containing 'pipeline' in their name."""
-    for file in os.listdir():
-        if "pipeline" in file and file.endswith(".json"):
-            os.remove(file)

pytest.ini

@@ -1,2 +1,5 @@
 [pytest]
+filterwarnings =
+    ignore:Import\(s\) unavailable to set up matplotlib support:UserWarning
+
 doctest_optionflags = NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL ELLIPSIS

setup.py

@@ -55,6 +55,7 @@ def read(filename):
         "pyyaml",
         "questionary",
         "randomname",
+        "semver",
         "rich-click",
         "streamlit",
         "tqdm",
@@ -73,6 +74,7 @@ def read(filename):
             "pytest",
             "pytest-asyncio",
             "pytest-cov",
+            "pytest-mock",
             "pytest-xdist",
             "sphinx",
             "sphinx_rtd_theme",
@@ -91,7 +93,7 @@ def read(filename):
     },
     include_package_data=True,
     package_data={
-        "pymorize": ["data/*.yaml"],
+        "pymorize": ["data/*.yaml", "data/cmip7/all_var_info.json"],
     },
     classifiers=[
         "Development Status :: 2 - Pre-Alpha",

src/pymorize/cmorizer.py

@@ -22,12 +22,10 @@
     set_dashboard_link,
 )
 from .config import PymorizeConfig, PymorizeConfigManager
-from .data_request import (
-    DataRequest,
-    DataRequestTable,
-    DataRequestVariable,
-    IgnoreTableFiles,
-)
+from .data_request.collection import DataRequest
+from .data_request.factory import create_factory
+from .data_request.table import DataRequestTable
+from .data_request.variable import DataRequestVariable
 from .filecache import fc
 from .logging import logger
 from .pipeline import Pipeline
@@ -39,9 +37,15 @@
 DIMENSIONLESS_MAPPING_TABLE = files("pymorize.data").joinpath(
     "dimensionless_mappings.yaml"
 )
+"""Path: The dimenionless unit mapping table, used to recreate meaningful units from
+dimensionless fractional values (e.g. 0.001 --> g/kg)"""
+# FIXME(PG): I don't know if this is a Path or not, so the documented type might be wrong
 
 
 class CMORizer:
+    _SUPPORTED_CMOR_VERSIONS = ("CMIP6", "CMIP7")
+    """tuple : Supported CMOR versions."""
+
     def __init__(
         self,
         pymorize_cfg=None,
@@ -61,6 +65,15 @@ def __init__(
         self.pipelines = pipelines_cfg or []
         self._cluster = None  # ask Cluster, might be set up later
         ################################################################################
+        # CMOR Version Settings:
+
+        if self._general_cfg.get("cmor_version") is None:
+            raise ValueError("cmor_version must be set in the general configuration.")
+        self.cmor_version = self._general_cfg["cmor_version"]
+        if self.cmor_version not in self._SUPPORTED_CMOR_VERSIONS:
+            logger.error(f"CMOR version {self.cmor_version} is not supported.")
+            logger.error(f"Supported versions are {self._SUPPORTED_CMOR_VERSION}")
+            raise ValueError(f"Unsupported CMOR version: {self.cmor_version}")
 
         ################################################################################
         # Print Out Configuration:
@@ -105,14 +118,19 @@ def __init__(
             logger.debug("...done!")
         self._post_init_create_pipelines()
         self._post_init_create_rules()
-        self._post_init_read_bare_tables()
+        self._post_init_create_data_request_tables()
         self._post_init_create_data_request()
         self._post_init_populate_rules_with_tables()
         self._post_init_read_dimensionless_unit_mappings()
         self._post_init_data_request_variables()
         logger.debug("...post-init done!")
         ################################################################################
 
+    def __del__(self):
+        """Gracefully close the cluster if it exists"""
+        if self._cluster is not None:
+            self._cluster.close()
+
     def _post_init_configure_dask(self):
         """
         Sets up configuration for Dask-Distributed
@@ -184,31 +202,26 @@ def _post_init_create_dask_cluster(self):
             import flox.xarray  # noqa: F401
         logger.info(f"...done! Imported {dask_extras} libraries.")
 
-    def _post_init_read_bare_tables(self):
+    def _post_init_create_data_request_tables(self):
         """
         Loads all the tables from table directory as a mapping object.
         A shortened version of the filename (i.e., ``CMIP6_Omon.json`` -> ``Omon``) is used as the mapping key.
         The same key format is used in CMIP6_table_id.json
         """
+        data_request_table_factory = create_factory(DataRequestTable)
+        DataRequestTableClass = data_request_table_factory.get(self.cmor_version)
         table_dir = Path(self._general_cfg["CMIP_Tables_Dir"])
-        table_files = {
-            path.stem.replace("CMIP6_", ""): path for path in table_dir.glob("*.json")
-        }
-        tables = {}
-        ignore_files = set(ignore_file.value for ignore_file in IgnoreTableFiles)
-        for tbl_name, tbl_file in table_files.items():
-            logger.debug(f"{tbl_name}, {tbl_file}")
-            if tbl_file.name not in ignore_files:
-                logger.debug(f"Adding Table {tbl_name}")
-                tables[tbl_name] = DataRequestTable(tbl_file)
+        tables = DataRequestTableClass.table_dict_from_directory(table_dir)
         self._general_cfg["tables"] = self.tables = tables
 
     def _post_init_create_data_request(self):
         """
         Creates a DataRequest object from the tables directory.
         """
         table_dir = self._general_cfg["CMIP_Tables_Dir"]
-        self.data_request = DataRequest.from_tables_dir(table_dir)
+        data_request_factory = create_factory(DataRequest)
+        DataRequestClass = data_request_factory.get(self.cmor_version)
+        self.data_request = DataRequestClass.from_directory(table_dir)
 
     def _post_init_populate_rules_with_tables(self):
         """
@@ -217,11 +230,11 @@ def _post_init_populate_rules_with_tables(self):
         tables = self._general_cfg["tables"]
         for rule in self.rules:
             for tbl in tables.values():
-                if rule.cmor_variable in tbl.variable_ids:
+                if rule.cmor_variable in tbl.variables:
                     rule.add_table(tbl.table_id)
 
     def _post_init_data_request_variables(self):
-        for drv in self.data_request.variables:
+        for drv in self.data_request.variables.values():
             rule_for_var = self.find_matching_rule(drv)
             if rule_for_var is None:
                 continue
@@ -312,10 +325,11 @@ def _rules_expand_drvs(self):
         self.rules = new_rules
 
     def _rules_depluralize_drvs(self):
+        """Ensures that only one data request variable is assigned to each rule"""
         for rule in self.rules:
             assert len(rule.data_request_variables) == 1
-            drv = rule.data_request_variable = rule.data_request_variables[0]
-            drv.depluralize()
+            rule.data_request_variable = rule.data_request_variables[0]
+            del rule.data_request_variables
 
     def _post_init_create_pipelines(self):
         pipelines = []
@@ -371,7 +385,7 @@ def _check_is_subperiod(self):
         errors = []
         for rule in self.rules:
             table_freq = _frequency_from_approx_interval(
-                rule.data_request_variable.table.approx_interval
+                rule.data_request_variable.table_header.approx_interval
             )
             # is_subperiod from pandas does not support YE or ME notation
             table_freq = table_freq.rstrip("E")