Cluster network by shapes (NUTS0, NUTS1, NUTS2, NUTS3, and ADM1)

config/config.default.yaml

@@ -870,6 +870,12 @@ costs:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#clustering
 clustering:
+  mode: admin # buses, admin, custom_admin_map, custom_busmap
+  admin:
+    level: 1 # Set general level of administrative division, e.g. 0 for countries, 1 for regions, specify individual countries if needed
+    DE: 3
+    FR: 2
+    ES: 0
   focus_weights: false
   simplify_network:
     to_substations: false

doc/data-bundle.rst

@@ -33,13 +33,6 @@ scope to reduce file size, or are not provided through stable URLs elsewhere.
 - **License:** `custom <https://ec.europa.eu/eurostat/about-us/policies/copyright>`__
 - **Description:** Average annual population to calculate regional GDP data (thousand persons) by NUTS 3 regions.
 
-``data/bundle/nama_10r_3gdp.tsv.gz``
-
-- **Source:** Eurostat
-- **Link:** https://ec.europa.eu/eurostat/databrowser/view/nama_10r_3gdp/default/table?lang=en
-- **License:** `custom <https://ec.europa.eu/eurostat/about-us/policies/copyright>`__
-- **Description:** Gross domestic product (GDP) at current market prices by NUTS 3 regions.
-
 ``data/bundle/corine``
 
 - **Source:** European Environment Agency (EEA)
@@ -94,7 +87,7 @@ scope to reduce file size, or are not provided through stable URLs elsewhere.
 - **License:** CC0 (`reference <https://datadryad.org/stash/dataset/doi:10.5061/dryad.dk1j0>`__)
 - **Description:** Gridded GDP data.
 
-``data/bundle/ppp_2013_1km_Aggregated.tif``
+``data/bundle/ppp_2019_1km_Aggregated.tif``
 
 - **Source:** WorldPop (www.worldpop.org - School of Geography and Environmental
   Science, University of Southampton; Department of Geography and Geosciences,
@@ -104,5 +97,5 @@ scope to reduce file size, or are not provided through stable URLs elsewhere.
   Funded by The Bill and Melinda Gates Foundation (OPP1134076).
   https://dx.doi.org/10.5258/SOTON/WP00647
 - **Link:** https://hub.worldpop.org/doi/10.5258/SOTON/WP00647
-- **License:** CC-BY 4.0 (`reference <https://hub.worldpop.org/geodata/summary?id=24770>`__)
+- **License:** CC-BY 4.0 (`reference <https://hub.worldpop.org/geodata/summary?id=24776>`__)
 - **Description:** Gridded population data.

doc/release_notes.rst

@@ -115,6 +115,8 @@ Upcoming Release
 
 * Update locations and capacities of ammonia plants.
 
+* Updating all base shapes (country_shapes, europe_shape, nuts3_shapes, ...). The workflow has been modified to use higher resolution and more harmonised shapes (NUTS3 2021 01M data and OSM administration level 1 for non-NUTS3 countries, such as BA, MD, UA, and XK). Data sources for population and GDP p.c. have been updated to JRC ARDECO https://urban.jrc.ec.europa.eu/ardeco/ -- 2019 values are used. `build_gdp_pop_non_nuts3` (originally created to build regional GDP p.c. and population data for MD and UA) is now integrated into `build_shapes` and extended to build regional values for all non-NUTS3 countries using cutouts of the updated datasets `GDP_per_capita_PPP_1990_2015_v2.nc` and `ppp_2019_1km_Aggregated.tif`,
+
 
 PyPSA-Eur 0.13.0 (13th September 2024)
 ======================================

rules/build_electricity.smk

@@ -98,17 +98,45 @@ rule base_network:
         "../scripts/base_network.py"
 
 
+rule build_osm_boundaries:
+    input:
+        json="data/osm-boundaries/json/{country}_adm1.json",
+        eez=ancient("data/eez/World_EEZ_v12_20231025_LR/eez_v12_lowres.gpkg"),
+    output:
+        boundary="data/osm-boundaries/build/{country}_adm1.geojson",
+    log:
+        "logs/build_osm_boundaries_{country}.log",
+    threads: 1
+    resources:
+        mem_mb=1500,
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/build_osm_boundaries.py"
+
+
+rule build_osm_boundaries_all:
+    input:
+        "data/osm-boundaries/build/MD_adm1.geojson",
+        "data/osm-boundaries/build/BA_adm1.geojson",
+        "data/osm-boundaries/build/UA_adm1.geojson",
+        "data/osm-boundaries/build/XK_adm1.geojson",
+
+
 rule build_shapes:
     params:
         countries=config_provider("countries"),
     input:
-        naturalearth=ancient("data/naturalearth/ne_10m_admin_0_countries_deu.shp"),
         eez=ancient("data/eez/World_EEZ_v12_20231025_LR/eez_v12_lowres.gpkg"),
-        nuts3=ancient("data/nuts/NUTS_RG_03M_2013_4326_LEVL_3.geojson"),
-        nuts3pop=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"),
-        nuts3gdp=ancient("data/bundle/nama_10r_3gdp.tsv.gz"),
-        ch_cantons=ancient("data/ch_cantons.csv"),
-        ch_popgdp=ancient("data/bundle/je-e-21.03.02.xls"),
+        nuts3_2021="data/nuts/NUTS_RG_01M_2021_4326_LEVL_3.geojson",
+        ba_adm1="data/osm-boundaries/build/BA_adm1.geojson",
+        md_adm1="data/osm-boundaries/build/MD_adm1.geojson",
+        ua_adm1="data/osm-boundaries/build/UA_adm1.geojson",
+        xk_adm1="data/osm-boundaries/build/XK_adm1.geojson",
+        nuts3_gdp="data/jrc-ardeco/ARDECO-SUVGDP.2021.table.csv",
+        nuts3_pop="data/jrc-ardeco/ARDECO-SNPTD.2021.table.csv",
+        other_gdp="data/sandbox/GDP_per_capita_PPP_1990_2015_v2.nc",  # TODO: update links to data/bundle after data bundle update
+        other_pop="data/sandbox/ppp_2019_1km_Aggregated.tif",  # TODO: update links to data/bundle after data bundle update
     output:
         country_shapes=resources("country_shapes.geojson"),
         offshore_shapes=resources("offshore_shapes.geojson"),
@@ -472,42 +500,10 @@ def input_conventional(w):
     }
 
 
-# Optional input when having Ukraine (UA) or Moldova (MD) in the countries list
-def input_gdp_pop_non_nuts3(w):
-    countries = set(config_provider("countries")(w))
-    if {"UA", "MD"}.intersection(countries):
-        return {"gdp_pop_non_nuts3": resources("gdp_pop_non_nuts3.geojson")}
-    return {}
-
-
-rule build_gdp_pop_non_nuts3:
-    params:
-        countries=config_provider("countries"),
-    input:
-        base_network=resources("networks/base_s.nc"),
-        regions=resources("regions_onshore_base_s.geojson"),
-        gdp_non_nuts3="data/bundle/GDP_per_capita_PPP_1990_2015_v2.nc",
-        pop_non_nuts3="data/bundle/ppp_2013_1km_Aggregated.tif",
-    output:
-        resources("gdp_pop_non_nuts3.geojson"),
-    log:
-        logs("build_gdp_pop_non_nuts3.log"),
-    benchmark:
-        benchmarks("build_gdp_pop_non_nuts3")
-    threads: 1
-    resources:
-        mem_mb=8000,
-    conda:
-        "../envs/environment.yaml"
-    script:
-        "../scripts/build_gdp_pop_non_nuts3.py"
-
-
 rule build_electricity_demand_base:
     params:
         distribution_key=config_provider("load", "distribution_key"),
     input:
-        unpack(input_gdp_pop_non_nuts3),
         base_network=resources("networks/base_s.nc"),
         regions=resources("regions_onshore_base_s.geojson"),
         nuts3=resources("nuts3_shapes.geojson"),
@@ -593,6 +589,8 @@ def input_cluster_network(w):
 
 rule cluster_network:
     params:
+        mode=config_provider("clustering", "mode"),
+        admin=config_provider("clustering", "admin"),
         cluster_network=config_provider("clustering", "cluster_network"),
         aggregation_strategies=config_provider(
             "clustering", "aggregation_strategies", default={}
@@ -608,9 +606,9 @@ rule cluster_network:
     input:
         unpack(input_cluster_network),
         network=resources("networks/base_s.nc"),
+        nuts3_shapes=resources("nuts3_shapes.geojson"),
         regions_onshore=resources("regions_onshore_base_s.geojson"),
         regions_offshore=resources("regions_offshore_base_s.geojson"),
-        busmap=ancient(resources("busmap_base_s.csv")),
         hac_features=lambda w: (
             resources("hac_features.nc")
             if config_provider("clustering", "cluster_network", "algorithm")(w)

rules/build_sector.smk

@@ -468,6 +468,9 @@ rule build_heat_totals:
         "../scripts/build_heat_totals.py"
 
 
+# TODO: update long-term to only need NUTS2021, nuts3_shapes.geojson
+# However: requires manual updating of NUTS code changes 2016 to 2021:
+# https://ec.europa.eu/eurostat/web/nuts/history
 rule build_biomass_potentials:
     params:
         biomass=config_provider("biomass"),

rules/retrieve.smk

@@ -18,16 +18,15 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle",
     datafiles = [
         "je-e-21.03.02.xls",
         "nama_10r_3popgdp.tsv.gz",
-        "nama_10r_3gdp.tsv.gz",
         "corine/g250_clc06_V18_5.tif",
         "eea/UNFCCC_v23.csv",
         "emobility/KFZ__count",
         "emobility/Pkw__count",
         "h2_salt_caverns_GWh_per_sqkm.geojson",
         "natura/natura.tiff",
         "gebco/GEBCO_2014_2D.nc",
-        "GDP_per_capita_PPP_1990_2015_v2.nc",
-        "ppp_2013_1km_Aggregated.tif",
+        "GDP_per_capita_PPP_1990_2015_v2.nc",  # TODO: update
+        "ppp_2013_1km_Aggregated.tif",  # TODO: update
     ]
 
     rule retrieve_databundle:
@@ -77,7 +76,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle",
 
 if config["enable"]["retrieve"]:
 
-    rule retrieve_nuts_shapes:
+    rule retrieve_nuts_2013_shapes:
         input:
             shapes=storage(
                 "https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2013-03m.geojson.zip"
@@ -101,6 +100,34 @@ if config["enable"]["retrieve"]:
 
 
 
+if config["enable"]["retrieve"]:
+
+    rule retrieve_nuts_2021_shapes:
+        input:
+            shapes=storage(
+                "https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2021-01m.geojson.zip"
+            ),
+        output:
+            shapes_level_3="data/nuts/NUTS_RG_01M_2021_4326_LEVL_3.geojson",
+            shapes_level_2="data/nuts/NUTS_RG_01M_2021_4326_LEVL_2.geojson",
+            shapes_level_1="data/nuts/NUTS_RG_01M_2021_4326_LEVL_1.geojson",
+            shapes_level_0="data/nuts/NUTS_RG_01M_2021_4326_LEVL_0.geojson",
+        params:
+            zip_file="data/nuts/ref-nuts-2021-01m.geojson.zip",
+        run:
+            os.rename(input.shapes, params.zip_file)
+            with ZipFile(params.zip_file, "r") as zip_ref:
+                for level in ["LEVL_3", "LEVL_2", "LEVL_1", "LEVL_0"]:
+                    filename = f"NUTS_RG_01M_2021_4326_{level}.geojson"
+                    zip_ref.extract(filename, Path(output.shapes_level_0).parent)
+                    extracted_file = Path(output.shapes_level_0).parent / filename
+                    extracted_file.rename(
+                        getattr(output, f"shapes_level_{level[-1]}")
+                    )
+            os.remove(params.zip_file)
+
+
+
 if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True):
 
     rule retrieve_cutout:
@@ -371,27 +398,6 @@ if config["enable"]["retrieve"]:
 
 
 
-if config["enable"]["retrieve"]:
-
-    # Download directly from naciscdn.org which is a redirect from naturalearth.com
-    # (https://www.naturalearthdata.com/downloads/10m-cultural-vectors/10m-admin-0-countries/)
-    # Use point-of-view (POV) variant of Germany so that Crimea is included.
-    rule retrieve_naturalearth_countries:
-        input:
-            storage(
-                "https://naciscdn.org/naturalearth/10m/cultural/ne_10m_admin_0_countries_deu.zip"
-            ),
-        params:
-            zip="data/naturalearth/ne_10m_admin_0_countries_deu.zip",
-        output:
-            countries="data/naturalearth/ne_10m_admin_0_countries_deu.shp",
-        run:
-            move(input[0], params["zip"])
-            output_folder = Path(output["countries"]).parent
-            unpack_archive(params["zip"], output_folder)
-            os.remove(params["zip"])
-
-
 if config["enable"]["retrieve"]:
 
     rule retrieve_gem_europe_gas_tracker:
@@ -629,6 +635,20 @@ if config["enable"]["retrieve"] and (
             ),
 
 
+if config["enable"]["retrieve"]:
+
+    rule retrieve_osm_boundaries:
+        output:
+            json="data/osm-boundaries/json/{country}_adm1.json",
+        log:
+            "logs/retrieve_osm_boundaries_{country}_adm1.log",
+        threads: 1
+        conda:
+            "../envs/retrieve.yaml"
+        script:
+            "../scripts/retrieve_osm_boundaries.py"
+
+
 if config["enable"]["retrieve"]:
 
     rule retrieve_heat_source_utilisation_potentials:
@@ -645,3 +665,48 @@ if config["enable"]["retrieve"]:
             "data/heat_source_utilisation_potentials/{heat_source}.gpkg",
         script:
             "../scripts/retrieve_heat_source_utilisation_potentials.py"
+
+
+if config["enable"]["retrieve"]:
+
+    rule retrieve_jrc_ardeco:
+        output:
+            ardeco_gdp="data/jrc-ardeco/ARDECO-SUVGDP.2021.table.csv",
+            ardeco_pop="data/jrc-ardeco/ARDECO-SNPTD.2021.table.csv",
+        run:
+            import requests
+
+            urls = {
+                "ardeco_gdp": "https://urban.jrc.ec.europa.eu/ardeco-api-v2/rest/export/SUVGDP?version=2021&format=csv-table",
+                "ardeco_pop": "https://urban.jrc.ec.europa.eu/ardeco-api-v2/rest/export/SNPTD?version=2021&format=csv-table",
+            }
+
+            for key, url in urls.items():
+                response = requests.get(url)
+                output_path = output[key] if key in urls else None
+                if output_path:
+                    with open(output_path, "wb") as f:
+                        f.write(response.content)
+
+
+
+# TODO: Remove before merging into master and after updating databundle on Zenodo:
+if config["enable"]["retrieve"]:
+
+    rule retrieve_sandbox_data:
+        input:
+            gdp_non_nuts3=storage(
+                "https://github.com/bobbyxng/sandbox/raw/refs/heads/main/data/GDP_per_capita_PPP_1990_2015_v2.nc",
+                keep_local=True,
+            ),
+            pop_non_nuts_3=storage(
+                "https://github.com/bobbyxng/sandbox/raw/refs/heads/main/data/ppp_2019_1km_Aggregated.tif",
+                keep_local=True,
+            ),
+        output:
+            gdp_non_nuts3="data/sandbox/GDP_per_capita_PPP_1990_2015_v2.nc",
+            pop_non_nuts_3="data/sandbox/ppp_2019_1km_Aggregated.tif",
+        retries: 1
+        run:
+            for key in input.keys():
+                move(input[key], output[key])

scripts/build_electricity_demand_base.py

@@ -44,7 +44,6 @@ def upsample_load(
     regions_fn: str,
     load_fn: str,
     nuts3_fn: str,
-    gdp_pop_non_nuts3_fn: str,
     distribution_key: dict[str, float],
 ) -> pd.DataFrame:
     substation_lv_i = n.buses.index[n.buses["substation_lv"]]
@@ -61,19 +60,7 @@ def upsample_load(
     for cntry, group in gdf_regions.geometry.groupby(gdf_regions.country):
         load_ct = load[cntry]
 
-        if cntry in ["UA", "MD"]:
-            # separate handling because nuts3 provides no data for UA+MD
-            gdp_pop_non_nuts3 = gpd.read_file(gdp_pop_non_nuts3_fn).set_index("Bus")
-            gdp_pop_non_nuts3 = gdp_pop_non_nuts3.loc[
-                (gdp_pop_non_nuts3.country == cntry)
-                & (gdp_pop_non_nuts3.index.isin(substation_lv_i))
-            ]
-            factors = normed(
-                gdp_weight * normed(gdp_pop_non_nuts3["gdp"])
-                + pop_weight * normed(gdp_pop_non_nuts3["pop"])
-            )
-
-        elif len(group) == 1:
+        if len(group) == 1:
             factors = pd.Series(1.0, index=group.index)
 
         else:
@@ -116,7 +103,6 @@ def upsample_load(
         regions_fn=snakemake.input.regions,
         load_fn=snakemake.input.load,
         nuts3_fn=snakemake.input.nuts3,
-        gdp_pop_non_nuts3_fn=snakemake.input.get("gdp_pop_non_nuts3"),
         distribution_key=params.distribution_key,
     )