Merge branch 'develop' into bugfix-509-new-table

OpenEnergyPlatform · Apr 23, 2024 · d10d558 · d10d558
2 parents 4fa7f75 + 463b38e
commit d10d558
Show file tree

Hide file tree

Showing 10 changed files with 88 additions and 177 deletions.
diff --git a/.github/workflows/ci-develop.yml b/.github/workflows/ci-develop.yml
@@ -1,4 +1,4 @@
-name: CI
+name: CI - Pytest Develop
 
 on:
   workflow_dispatch:
@@ -20,9 +20,9 @@ jobs:
 
     # Configure tests
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

diff --git a/.github/workflows/ci-production.yml b/.github/workflows/ci-production.yml
@@ -1,4 +1,4 @@
-name: CI
+name: CI - Pytest Production
 
 on:
   workflow_dispatch:
@@ -20,9 +20,9 @@ jobs:
 
     # Configure tests
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,8 +10,10 @@ and the versioning aims to respect [Semantic Versioning](http://semver.org/spec/
 ## [v0.XX.X] - 202X-XX-XX
 ### Added
 - Add new table `changed_dso_assignment` [#510](https://github.com/OpenEnergyPlatform/open-MaStR/pull/510)
+- Add deprecation warning for `MaStRMirror` and `MaStRDownload` [#492](https://github.com/OpenEnergyPlatform/open-MaStR/pull/492)
 ### Changed
 ### Removed
+- Remove outdated and deprecated dependencies [#506](https://github.com/OpenEnergyPlatform/open-MaStR/pull/506)
 
 ## [v0.14.2] Maintenance - 2024-04-10
 ### Changed
@@ -23,6 +25,7 @@ and the versioning aims to respect [Semantic Versioning](http://semver.org/spec/
 ### Removed
 - Remove unused Docker File [#501](https://github.com/OpenEnergyPlatform/open-MaStR/pull/501)
 
+
 ## [v0.14.1] Hotfix - 2024-01-17
 ### Changed
 - Change data type of NetzbetreiberpruefungStatus to string [#483](https://github.com/OpenEnergyPlatform/open-MaStR/pull/483)

diff --git a/open_mastr/mastr.py b/open_mastr/mastr.py
@@ -54,6 +54,15 @@ class Mastr:
     A SQL database is used to mirror the MaStR database. It can be filled with
     data either from the MaStR-bulk download or from the MaStR-API.
 
+    !!! example
+
+        ```python
+        from open_mastr import Mastr
+
+        db = Mastr()
+        db.download()
+        ```
+
     Parameters
     ----------
     engine : {'sqlite', sqlalchemy.engine.Engine}, optional
@@ -63,13 +72,8 @@ class Mastr:
             Allows connection to an existing translated database. Default is 'False'.
             Only for 'sqlite'-type engines.
 
-    !!! example
-        ```python
-        from open_mastr import Mastr
 
-        db = Mastr()
-        db.download()
-        ```
+
     """
 
     def __init__(self, engine="sqlite", connect_to_translated_db=False) -> None:

diff --git a/open_mastr/soap_api/download.py b/open_mastr/soap_api/download.py
@@ -414,7 +414,14 @@ def _missed_units_to_file(data, data_type, missed_units):
 
 
 class MaStRDownload:
-    """Use the higher level interface for bulk download
+    """
+    !!! warning
+
+        **This class is deprecated** and will not be maintained from version 0.15.0 onwards.
+        Instead use [`Mastr.download`][open_mastr.Mastr.download] with parameter
+        `method` = "bulk" to get bulk downloads of the dataset.
+
+    Use the higher level interface for bulk download
 
     `MaStRDownload` builds on top of [`MaStRAPI`][open_mastr.soap_api.download.MaStRAPI] and provides
     an interface for easier downloading.
@@ -451,6 +458,17 @@ def __init__(self, parallel_processes=None):
             multiprocessing package) choose False.
             Defaults to number of cores (including hyperthreading).
         """
+        log.warn(
+            """
+            The `MaStRDownload` class is deprecated and will not be maintained in the future.
+            To get a full table of the Marktstammdatenregister, use the open_mastr.Mastr.download
+            method.
+
+            If this change causes problems for you, please comment in this issue on github:
+            https://github.com/OpenEnergyPlatform/open-MaStR/issues/487
+
+            """
+        )
 
         # Number of parallel processes
         if parallel_processes == "max":
@@ -762,26 +780,30 @@ def basic_unit_data(self, data=None, limit=2000, date_from=None, max_retries=3):
             log.info(
                 f"Get list of units with basic information for data type {data} ({et})"
             )
-            yield from basic_data_download(
-                self._mastr_api,
-                "GetListeAlleEinheiten",
-                "Einheiten",
-                chunks_start,
-                limits,
-                date_from,
-                max_retries,
-                data,
-                et=et,
-            ) if et is None else basic_data_download(
-                self._mastr_api,
-                "GetGefilterteListeStromErzeuger",
-                "Einheiten",
-                chunks_start,
-                limits,
-                date_from,
-                max_retries,
-                data,
-                et=et,
+            yield from (
+                basic_data_download(
+                    self._mastr_api,
+                    "GetListeAlleEinheiten",
+                    "Einheiten",
+                    chunks_start,
+                    limits,
+                    date_from,
+                    max_retries,
+                    data,
+                    et=et,
+                )
+                if et is None
+                else basic_data_download(
+                    self._mastr_api,
+                    "GetGefilterteListeStromErzeuger",
+                    "Einheiten",
+                    chunks_start,
+                    limits,
+                    date_from,
+                    max_retries,
+                    data,
+                    et=et,
+                )
             )
 
     def additional_data(self, data, unit_ids, data_fcn, timeout=10):

diff --git a/open_mastr/soap_api/mirror.py b/open_mastr/soap_api/mirror.py
@@ -21,6 +21,12 @@
 
 class MaStRMirror:
     """
+    !!! warning
+
+        **This class is deprecated** and will not be maintained from version 0.15.0 onwards.
+        Instead use [`Mastr.download`][open_mastr.Mastr.download] with parameter
+        `method` = "bulk" to mirror the MaStR dataset to a local database.
+
     Mirror the Marktstammdatenregister database and keep it up-to-date.
 
     A PostgreSQL database is used to mirror the MaStR database. It builds
@@ -93,6 +99,18 @@ def __init__(
             Number of parallel processes used to download additional data.
             Defaults to `None`.
         """
+        log.warn(
+            """
+            The `MaStRMirror` class is deprecated and will not be maintained in the future.
+            To get a full table of the Marktstammdatenregister, use the open_mastr.Mastr.download
+            method.
+
+            If this change causes problems for you, please comment in this issue on github:
+            https://github.com/OpenEnergyPlatform/open-MaStR/issues/487
+
+            """
+        )
+
         self._engine = engine
 
         # Associate downloader
@@ -979,6 +997,7 @@ def restore(self, dumpfile):
 
 
         !!! warning
+
             If tables that are restored from the dump contain data, restore doesn't work!
 
         """

diff --git a/open_mastr/utils/orm.py b/open_mastr/utils/orm.py
@@ -1,5 +1,4 @@
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.schema import MetaData
+from sqlalchemy.orm import DeclarativeBase
 from sqlalchemy import (
     Column,
     Integer,
@@ -13,8 +12,9 @@
     JSON,
 )
 
-meta = MetaData()
-Base = declarative_base(metadata=meta)
+
+class Base(DeclarativeBase):
+    pass
 
 
 class ParentAllTables(object):

diff --git a/setup.py b/setup.py
@@ -45,13 +45,12 @@
     install_requires=[
         "pandas>=2.1",  # pandas 2.1 is needed for dataframe.map()
         "numpy",
-        "sqlalchemy",
+        "sqlalchemy>=2.0",
         "psycopg2-binary",
         "zeep",
         "tqdm",
         "requests",
         "keyring",
-        "tqdm",
         "pyyaml",
         "xmltodict",
     ],

diff --git a/tests/soap_api/test_mastr_mirror.py b/tests/soap_api/test_mastr_mirror.py
diff --git a/tests/xml_download/test_utils_write_to_database.py b/tests/xml_download/test_utils_write_to_database.py
@@ -157,10 +157,10 @@ def test_add_table_to_database(zipped_xml_file_path, engine_testdb):
 def test_add_zero_as_first_character_for_too_short_string():
     # Prepare
     df_raw = pd.DataFrame(
-        {"ID": [0, 1, 2], "Gemeindeschluessel": [9162000, None, 19123456]}
+        {"ID": [0, 1, 2], "Gemeindeschluessel": [9162000, np.nan, 19123456]}
     )
     df_correct = pd.DataFrame(
-        {"ID": [0, 1, 2], "Gemeindeschluessel": ["09162000", None, "19123456"]}
+        {"ID": [0, 1, 2], "Gemeindeschluessel": ["09162000", np.nan, "19123456"]}
     )
 
     # Act
@@ -236,5 +236,3 @@ def test_cast_date_columns_to_datetime():
     pd.testing.assert_frame_equal(
         df_replaced, cast_date_columns_to_datetime("anlageneegwasser", df_raw)
     )
-
-