From 8fdad39a2a8523a1de2d33379d8aec70a18558c8 Mon Sep 17 00:00:00 2001
From: Florian Kotthoff <74312290+FlorianK13@users.noreply.github.com>
Date: Fri, 8 Mar 2024 11:39:10 +0100
Subject: [PATCH 01/13] Update python version #487

---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 764b0476..821c6237 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -3,4 +3,4 @@ repos:
     rev: 22.6.0
     hooks:
       - id: black
-        language_version: python3.10
+        language_version: python3.11

From 491d22319ac3b56b595d1579cf5d313b19558bde Mon Sep 17 00:00:00 2001
From: Florian Kotthoff <74312290+FlorianK13@users.noreply.github.com>
Date: Fri, 8 Mar 2024 11:39:21 +0100
Subject: [PATCH 02/13] Add deprecation warning to MaStRDownload #487

---
 open_mastr/soap_api/download.py | 64 ++++++++++++++++++++++-----------
 1 file changed, 43 insertions(+), 21 deletions(-)

diff --git a/open_mastr/soap_api/download.py b/open_mastr/soap_api/download.py
index 6d74918d..04dbdb25 100644
--- a/open_mastr/soap_api/download.py
+++ b/open_mastr/soap_api/download.py
@@ -414,7 +414,14 @@ def _missed_units_to_file(data, data_type, missed_units):
 
 
 class MaStRDownload:
-    """Use the higher level interface for bulk download
+    """
+    !!! warning
+
+        **This class is deprecated** and will not be maintained from version 0.15.0 onwards.
+        Instead use [`Mastr.download`][open_mastr.Mastr.download] with parameter
+        `method` = "bulk" to get bulk downloads of the dataset.
+
+    Use the higher level interface for bulk download
 
     `MaStRDownload` builds on top of [`MaStRAPI`][open_mastr.soap_api.download.MaStRAPI] and provides
     an interface for easier downloading.
@@ -451,6 +458,17 @@ def __init__(self, parallel_processes=None):
             multiprocessing package) choose False.
             Defaults to number of cores (including hyperthreading).
         """
+        log.warn(
+            """
+            The `MaStRDownload` class is deprecated and will not be maintained in the future.
+            To get a full table of the Marktstammdatenregister, use the open_mastr.Mastr.download
+            method.
+
+            If this change causes problems for you, please comment in this issue on github:
+            https://github.com/OpenEnergyPlatform/open-MaStR/issues/487
+
+            """
+        )
 
         # Number of parallel processes
         if parallel_processes == "max":
@@ -762,26 +780,30 @@ def basic_unit_data(self, data=None, limit=2000, date_from=None, max_retries=3):
             log.info(
                 f"Get list of units with basic information for data type {data} ({et})"
             )
-            yield from basic_data_download(
-                self._mastr_api,
-                "GetListeAlleEinheiten",
-                "Einheiten",
-                chunks_start,
-                limits,
-                date_from,
-                max_retries,
-                data,
-                et=et,
-            ) if et is None else basic_data_download(
-                self._mastr_api,
-                "GetGefilterteListeStromErzeuger",
-                "Einheiten",
-                chunks_start,
-                limits,
-                date_from,
-                max_retries,
-                data,
-                et=et,
+            yield from (
+                basic_data_download(
+                    self._mastr_api,
+                    "GetListeAlleEinheiten",
+                    "Einheiten",
+                    chunks_start,
+                    limits,
+                    date_from,
+                    max_retries,
+                    data,
+                    et=et,
+                )
+                if et is None
+                else basic_data_download(
+                    self._mastr_api,
+                    "GetGefilterteListeStromErzeuger",
+                    "Einheiten",
+                    chunks_start,
+                    limits,
+                    date_from,
+                    max_retries,
+                    data,
+                    et=et,
+                )
             )
 
     def additional_data(self, data, unit_ids, data_fcn, timeout=10):

From e2f6d3f9df5949317c62b0afdee54c1f54c20f35 Mon Sep 17 00:00:00 2001
From: Florian Kotthoff <74312290+FlorianK13@users.noreply.github.com>
Date: Fri, 8 Mar 2024 12:01:36 +0100
Subject: [PATCH 03/13] Add deprecation warning for MaStRMirror class #487

Additionally, black did reformatting.
---
 open_mastr/soap_api/mirror.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/open_mastr/soap_api/mirror.py b/open_mastr/soap_api/mirror.py
index 379a1f83..9dda3c6e 100644
--- a/open_mastr/soap_api/mirror.py
+++ b/open_mastr/soap_api/mirror.py
@@ -21,6 +21,12 @@
 
 class MaStRMirror:
     """
+    !!! warning
+
+        **This class is deprecated** and will not be maintained from version 0.15.0 onwards.
+        Instead use [`Mastr.download`][open_mastr.Mastr.download] with parameter
+        `method` = "bulk" to mirror the MaStR dataset to a local database.
+
     Mirror the Marktstammdatenregister database and keep it up-to-date.
 
     A PostgreSQL database is used to mirror the MaStR database. It builds
@@ -93,6 +99,18 @@ def __init__(
             Number of parallel processes used to download additional data.
             Defaults to `None`.
         """
+        log.warn(
+            """
+            The `MaStRMirror` class is deprecated and will not be maintained in the future.
+            To get a full table of the Marktstammdatenregister, use the open_mastr.Mastr.download
+            method.
+
+            If this change causes problems for you, please comment in this issue on github:
+            https://github.com/OpenEnergyPlatform/open-MaStR/issues/487
+
+            """
+        )
+
         self._engine = engine
 
         # Associate downloader
@@ -979,6 +997,7 @@ def restore(self, dumpfile):
 
 
         !!! warning
+
             If tables that are restored from the dump contain data, restore doesn't work!
 
         """

From 5ccfb10839002ee926b1c623b89ada2a70132df1 Mon Sep 17 00:00:00 2001
From: Florian Kotthoff <74312290+FlorianK13@users.noreply.github.com>
Date: Fri, 8 Mar 2024 12:02:06 +0100
Subject: [PATCH 04/13] Correct docstring for mkdocs #487

Not related to this feature, but had to be done anyway.
---
 open_mastr/mastr.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/open_mastr/mastr.py b/open_mastr/mastr.py
index 329f4adf..acd626b8 100644
--- a/open_mastr/mastr.py
+++ b/open_mastr/mastr.py
@@ -54,6 +54,15 @@ class Mastr:
     A SQL database is used to mirror the MaStR database. It can be filled with
     data either from the MaStR-bulk download or from the MaStR-API.
 
+    !!! example
+
+        ```python
+        from open_mastr import Mastr
+
+        db = Mastr()
+        db.download()
+        ```
+
     Parameters
     ----------
     engine : {'sqlite', sqlalchemy.engine.Engine}, optional
@@ -63,13 +72,8 @@ class Mastr:
             Allows connection to an existing translated database. Default is 'False'.
             Only for 'sqlite'-type engines.
 
-    !!! example
-        ```python
-        from open_mastr import Mastr
 
-        db = Mastr()
-        db.download()
-        ```
+
     """
 
     def __init__(self, engine="sqlite", connect_to_translated_db=False) -> None:

From 9556311fd10763c280503f7f52b749386c560441 Mon Sep 17 00:00:00 2001
From: Florian Kotthoff <74312290+FlorianK13@users.noreply.github.com>
Date: Tue, 19 Mar 2024 15:11:42 +0100
Subject: [PATCH 05/13] Update Changelog #487

---
 CHANGELOG.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c5105412..7de1e8fb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,8 +8,9 @@ and the versioning aims to respect [Semantic Versioning](http://semver.org/spec/
 
 ## [v0.XX.X] unreleased - 2024-XX-XX
 ### Added
+- Add deprecation warning for `MaStRMirror` and `MaStRDownload` [#492](https://github.com/OpenEnergyPlatform/open-MaStR/pull/492)
 ### Changed
-- Fix and add URLs of example projects in readme [#481]((https://github.com/OpenEnergyPlatform/open-MaStR/pull/481)
+- Fix and add URLs of example projects in readme [#481](https://github.com/OpenEnergyPlatform/open-MaStR/pull/481)
 ### Removed
 
 ## [v0.14.1] Hotfix - 2024-01-17

From 77a52efb8aa819e3b6e8b89103aac583409c44c0 Mon Sep 17 00:00:00 2001
From: Florian Kotthoff <74312290+FlorianK13@users.noreply.github.com>
Date: Thu, 4 Apr 2024 10:32:49 +0200
Subject: [PATCH 06/13] Change dtype of None #505

Otherwise the pd.testing.assert_frame_equal
will throw an error in future versions.
---
 tests/xml_download/test_utils_write_to_database.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tests/xml_download/test_utils_write_to_database.py b/tests/xml_download/test_utils_write_to_database.py
index f28677e5..dc5aaacd 100644
--- a/tests/xml_download/test_utils_write_to_database.py
+++ b/tests/xml_download/test_utils_write_to_database.py
@@ -157,10 +157,10 @@ def test_add_table_to_database(zipped_xml_file_path, engine_testdb):
 def test_add_zero_as_first_character_for_too_short_string():
     # Prepare
     df_raw = pd.DataFrame(
-        {"ID": [0, 1, 2], "Gemeindeschluessel": [9162000, None, 19123456]}
+        {"ID": [0, 1, 2], "Gemeindeschluessel": [9162000, np.nan, 19123456]}
     )
     df_correct = pd.DataFrame(
-        {"ID": [0, 1, 2], "Gemeindeschluessel": ["09162000", None, "19123456"]}
+        {"ID": [0, 1, 2], "Gemeindeschluessel": ["09162000", np.nan, "19123456"]}
     )
 
     # Act
@@ -236,5 +236,3 @@ def test_cast_date_columns_to_datetime():
     pd.testing.assert_frame_equal(
         df_replaced, cast_date_columns_to_datetime("anlageneegwasser", df_raw)
     )
-
-

From f350abe53bccfd0866582fed931c74fcf4c071ff Mon Sep 17 00:00:00 2001
From: Florian Kotthoff <74312290+FlorianK13@users.noreply.github.com>
Date: Thu, 4 Apr 2024 12:11:35 +0200
Subject: [PATCH 07/13] Delete duplicated dependency #505

---
 setup.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/setup.py b/setup.py
index 32649986..4e209938 100644
--- a/setup.py
+++ b/setup.py
@@ -51,7 +51,6 @@
         "tqdm",
         "requests",
         "keyring",
-        "tqdm",
         "pyyaml",
         "xmltodict",
     ],

From 96769524d5da1b1b78ee8911231e93fd933f4e3c Mon Sep 17 00:00:00 2001
From: Florian Kotthoff <74312290+FlorianK13@users.noreply.github.com>
Date: Fri, 5 Apr 2024 15:25:25 +0200
Subject: [PATCH 08/13] Update orm import from sqlalchemy #505

---
 open_mastr/utils/orm.py | 4 +++-
 setup.py                | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/open_mastr/utils/orm.py b/open_mastr/utils/orm.py
index 5225f2cb..49d488aa 100644
--- a/open_mastr/utils/orm.py
+++ b/open_mastr/utils/orm.py
@@ -1,4 +1,4 @@
-from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import declarative_base
 from sqlalchemy.schema import MetaData
 from sqlalchemy import (
     Column,
@@ -204,6 +204,7 @@ class BiomassExtended(Extended, ParentAllTables, Base):
     EegMastrNummer = Column(String)
     KwkMastrNummer = Column(String)
 
+
 class CombustionExtended(Extended, ParentAllTables, Base):
     __tablename__ = "combustion_extended"
 
@@ -228,6 +229,7 @@ class CombustionExtended(Extended, ParentAllTables, Base):
     Technologie = Column(String)
     AusschliesslicheVerwendungImKombibetrieb = Column(Boolean)
 
+
 class GsgkExtended(Extended, ParentAllTables, Base):
     __tablename__ = "gsgk_extended"
 
diff --git a/setup.py b/setup.py
index 4e209938..6565f25c 100644
--- a/setup.py
+++ b/setup.py
@@ -45,7 +45,7 @@
     install_requires=[
         "pandas>=2.1",  # pandas 2.1 is needed for dataframe.map()
         "numpy",
-        "sqlalchemy",
+        "sqlalchemy>=2.0",
         "psycopg2-binary",
         "zeep",
         "tqdm",

From 6712875eed7f9d346b809dcd05209c3c7c4ab8ea Mon Sep 17 00:00:00 2001
From: Florian Kotthoff <74312290+FlorianK13@users.noreply.github.com>
Date: Tue, 9 Apr 2024 17:23:53 +0200
Subject: [PATCH 09/13] Update to sqlalchemy DeclarativeBase #505

---
 open_mastr/utils/orm.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/open_mastr/utils/orm.py b/open_mastr/utils/orm.py
index 49d488aa..2a25be7a 100644
--- a/open_mastr/utils/orm.py
+++ b/open_mastr/utils/orm.py
@@ -1,5 +1,4 @@
-from sqlalchemy.orm import declarative_base
-from sqlalchemy.schema import MetaData
+from sqlalchemy.orm import DeclarativeBase
 from sqlalchemy import (
     Column,
     Integer,
@@ -13,8 +12,9 @@
     JSON,
 )
 
-meta = MetaData()
-Base = declarative_base(metadata=meta)
+
+class Base(DeclarativeBase):
+    pass
 
 
 class ParentAllTables(object):

From 34567bed0b72b3468c4235a4b01f3f3a3bf6bd39 Mon Sep 17 00:00:00 2001
From: Florian Kotthoff <74312290+FlorianK13@users.noreply.github.com>
Date: Tue, 9 Apr 2024 17:28:41 +0200
Subject: [PATCH 10/13] Update github action versions #505

---
 .github/workflows/ci-develop.yml    | 4 ++--
 .github/workflows/ci-production.yml | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci-develop.yml b/.github/workflows/ci-develop.yml
index db2d3848..d22f20c4 100644
--- a/.github/workflows/ci-develop.yml
+++ b/.github/workflows/ci-develop.yml
@@ -20,9 +20,9 @@ jobs:
 
     # Configure tests
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
diff --git a/.github/workflows/ci-production.yml b/.github/workflows/ci-production.yml
index d73609ea..bb7f80d4 100644
--- a/.github/workflows/ci-production.yml
+++ b/.github/workflows/ci-production.yml
@@ -20,9 +20,9 @@ jobs:
 
     # Configure tests
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

From b3418d88b5b100b71eefda70b865bfa50980a4e5 Mon Sep 17 00:00:00 2001
From: Florian Kotthoff <74312290+FlorianK13@users.noreply.github.com>
Date: Tue, 9 Apr 2024 17:29:40 +0200
Subject: [PATCH 11/13] Rename workflows #505

So that they can be distinguished when looking at the github actions.
---
 .github/workflows/ci-develop.yml    | 2 +-
 .github/workflows/ci-production.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci-develop.yml b/.github/workflows/ci-develop.yml
index d22f20c4..99fb8c65 100644
--- a/.github/workflows/ci-develop.yml
+++ b/.github/workflows/ci-develop.yml
@@ -1,4 +1,4 @@
-name: CI
+name: CI - Pytest Develop
 
 on:
   workflow_dispatch:
diff --git a/.github/workflows/ci-production.yml b/.github/workflows/ci-production.yml
index bb7f80d4..2c118ffc 100644
--- a/.github/workflows/ci-production.yml
+++ b/.github/workflows/ci-production.yml
@@ -1,4 +1,4 @@
-name: CI
+name: CI - Pytest Production
 
 on:
   workflow_dispatch:

From 3740a50157bdde4b6985e6ba79462e5e42058e9c Mon Sep 17 00:00:00 2001
From: Florian Kotthoff <74312290+FlorianK13@users.noreply.github.com>
Date: Wed, 10 Apr 2024 08:50:48 +0200
Subject: [PATCH 12/13] Update Changelog #505

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 04115f7d..17cf166d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,7 @@ and the versioning aims to respect [Semantic Versioning](http://semver.org/spec/
 - Implement relevant API WSDL Patchnotes V24.1.128 [#499](https://github.com/OpenEnergyPlatform/open-MaStR/pull/499)
 ### Removed
 - Remove unused Docker File [#501](https://github.com/OpenEnergyPlatform/open-MaStR/pull/501)
+- Remove outdated and deprecated dependencies [#506](https://github.com/OpenEnergyPlatform/open-MaStR/pull/506)
 
 ## [v0.14.1] Hotfix - 2024-01-17
 ### Changed

From 51cbd9a4d35e233c3e3dd4b7892ff14bd452b508 Mon Sep 17 00:00:00 2001
From: Florian Kotthoff <74312290+FlorianK13@users.noreply.github.com>
Date: Tue, 23 Apr 2024 08:25:25 +0200
Subject: [PATCH 13/13] Delte tests that are not working #487

---
 tests/soap_api/test_mastr_mirror.py | 134 ----------------------------
 1 file changed, 134 deletions(-)
 delete mode 100644 tests/soap_api/test_mastr_mirror.py

diff --git a/tests/soap_api/test_mastr_mirror.py b/tests/soap_api/test_mastr_mirror.py
deleted file mode 100644
index fcca9fd0..00000000
--- a/tests/soap_api/test_mastr_mirror.py
+++ /dev/null
@@ -1,134 +0,0 @@
-import datetime
-import pytest
-import random
-import pandas as pd
-from os.path import join
-
-import pytest
-from open_mastr.soap_api.mirror import MaStRMirror
-from open_mastr.utils import orm
-from open_mastr.utils.config import get_project_home_dir
-from open_mastr.utils.constants import API_DATA_TYPES, API_LOCATION_TYPES
-from open_mastr.utils.helpers import create_database_engine, session_scope
-
-random_technologies = random.sample(
-    [
-        "wind",
-        "hydro",
-        "solar",
-        "biomass",
-        "nuclear",
-        "gsgk",
-        "storage",
-    ],
-    k=3,
-)
-
-LIMIT = 1
-DATE = datetime.datetime(2020, 11, 27, 0, 0, 0)
-
-
-@pytest.fixture
-def mastr_mirror():
-    engine = create_database_engine(
-        "sqlite", join(get_project_home_dir(), "data", "sqlite")
-    )
-    return MaStRMirror(engine=engine)
-
-
-@pytest.mark.dependency(name="backfill_basic")
-def test_backfill_basic(mastr_mirror, engine):
-    mastr_mirror.backfill_basic(data=random_technologies, date=DATE, limit=LIMIT)
-
-    # The table basic_units should have at least as many rows as TECHNOLOGIES were queried
-    with session_scope(engine=engine) as session:
-        response = session.query(orm.BasicUnit).count()
-        assert response >= len(random_technologies)
-
-
-@pytest.mark.dependency(depends=["backfill_basic"], name="retrieve_additional_data")
-def test_retrieve_additional_data(mastr_mirror):
-    for tech in random_technologies:
-        for data_type in API_DATA_TYPES:
-            mastr_mirror.retrieve_additional_data(
-                data=tech, data_type=data_type, limit=10 * LIMIT
-            )
-
-
-@pytest.mark.dependency(depends=["retrieve_additional_data"], name="update_latest")
-def test_update_latest(mastr_mirror, engine):
-    mastr_mirror.backfill_basic(data=random_technologies, date="latest", limit=LIMIT)
-
-    # Test if latest date is newer that initially requested data in backfill_basic
-    with session_scope(engine=engine) as session:
-        response = (
-            session.query(orm.BasicUnit.DatumLetzteAktualisierung)
-            .order_by(orm.BasicUnit.DatumLetzteAktualisierung.desc())
-            .first()
-        )
-    assert response.DatumLetzteAktualisierung > DATE
-
-
-# @pytest.mark.dependency(
-#     depends=["update_latest"], name="create_additional_data_requests"
-# )
-# def test_create_additional_data_requests(mastr_mirror, engine):
-#     with session_scope(engine=engine) as session:
-#         for tech in random_technologies:
-#             session.query(orm.AdditionalDataRequested).filter_by(
-#                 technology="gsgk"
-#             ).delete()
-#             session.commit()
-#             mastr_mirror.create_additional_data_requests(
-#                 tech, data_types=API_DATA_TYPES
-#             )
-
-
-@pytest.mark.dependency(name="backfill_locations_basic")
-def test_backfill_locations_basic(mastr_mirror, engine):
-    with session_scope(engine=engine) as session:
-        rows_before_download = session.query(orm.LocationBasic).count()
-
-    mastr_mirror.backfill_locations_basic(date="latest", limit=LIMIT)
-
-    # The table locations_basic should have rows_before_download + LIMIT rows
-    with session_scope(engine=engine) as session:
-        rows_after_download = session.query(orm.LocationBasic).count()
-        rows_downloaded = rows_after_download - rows_before_download
-        # Downloaded rows might already exist, therefore less or equal
-        assert rows_downloaded <= LIMIT
-
-
-@pytest.mark.dependency(
-    depends=["backfill_locations_basic"], name="test_retrieve_additional_location_data"
-)
-def test_retrieve_additional_location_data(mastr_mirror):
-    """Test if code runs successfully"""
-    for location_type in API_LOCATION_TYPES:
-        mastr_mirror.retrieve_additional_location_data(location_type, limit=LIMIT)
-
-
-def test_append_additional_data_from_basic_unit(mastr_mirror):
-
-    data_list = []
-    basic_unit = {
-        "EinheitMastrNummer": "SEE946206606199",
-        "Einheitart": "Stromerzeugungseinheit",
-        "Einheittyp": "Windeinheit",
-        "EegMastrNummer": "EEG993769703803",
-        "KwkMastrNummer": None,
-        "GenMastrNummer": "SGE924412236812",
-    }
-
-    for basic_unit_identifier, data_type in [
-        ("EinheitMastrNummer", "unit_data"),
-        ("EegMastrNummer", "eeg_data"),
-        ("KwkMastrNummer", "kwk_data"),
-        ("GenMastrNummer", "permit_data"),
-    ]:
-        data_list = mastr_mirror._append_additional_data_from_basic_unit(
-            data_list, basic_unit, basic_unit_identifier, data_type
-        )
-        assert type(data_list) == list
-        if data_type != "kwk_data":
-            assert len(data_list) > 0