From 97c2d16d04f2c1c2e265b44bc828823ce8cec5b7 Mon Sep 17 00:00:00 2001 From: "Bernat, Anastasia" Date: Mon, 7 Oct 2024 16:14:51 -0700 Subject: [PATCH] Testing deployment pipeline. --- README.md | 21 +- dash_app/app.py | 99 ++----- dash_app/assets/map.css | 4 + dash_app/definitions.py | 67 ++--- dash_app/holoview_plot.html | 66 ----- dash_app/holoview_plot2.html | 66 ----- dash_app/layout.py | 321 +++++++--------------- dash_app/other.py | 15 -- dash_app/run.py | 18 -- dash_app/src/datashader_holoviews.py | 130 --------- dash_app/src/datashader_mapbox.py | 103 ------- dash_app/src/deckgl.py | 388 +++++++++------------------ dash_app/src/dir2yaml.py | 2 - dash_app/src/imshow.py | 57 ---- dash_app/src/layer_maker.py | 75 ------ dash_app/src/leaflet_titiler.py | 189 ------------- dash_app/src/mapbox_raster.py | 280 ------------------- dash_app/src/reader.py | 112 -------- dash_app/src/reader_deckgl.py | 287 -------------------- dash_app/src/utilities.py | 198 +------------- dash_app/test.py | 169 ------------ dash_app/test2.py | 17 -- dash_app/test3.py | 95 ------- dash_app/test4.py | 81 ------ lambda_function.py | 13 +- requirements.txt | 4 + 26 files changed, 293 insertions(+), 2584 deletions(-) delete mode 100644 dash_app/holoview_plot.html delete mode 100644 dash_app/holoview_plot2.html delete mode 100644 dash_app/other.py delete mode 100644 dash_app/run.py delete mode 100644 dash_app/src/datashader_holoviews.py delete mode 100644 dash_app/src/datashader_mapbox.py delete mode 100644 dash_app/src/imshow.py delete mode 100644 dash_app/src/layer_maker.py delete mode 100644 dash_app/src/leaflet_titiler.py delete mode 100644 dash_app/src/mapbox_raster.py delete mode 100644 dash_app/src/reader_deckgl.py delete mode 100644 dash_app/test.py delete mode 100644 dash_app/test2.py delete mode 100644 dash_app/test3.py delete mode 100644 dash_app/test4.py diff --git a/README.md b/README.md index 0e8327b..5ea8505 100644 --- a/README.md +++ b/README.md @@ -66,19 +66,28 @@ Also, GRIDCERF most importantly requires **Dash version >=2.9.2**. Dash >=2.9.2 The `requirements.txt` in `dash_app` lists Python packages and their versions needed to run GRIDCERF following Python installation. We suggest your environment reflects these package versions. -![](https://img.shields.io/badge/bokeh-3.3.4-darkgrey) +![](https://img.shields.io/badge/boto3-1.34.112-darkgrey) +![](https://img.shields.io/badge/pydeck-0.9.1-darkgrey) ![](https://img.shields.io/badge/dash-2.14.2-darkgrey) ![](https://img.shields.io/badge/dash_bootstrap_components-1.5.0-darkgrey) -![](https://img.shields.io/badge/dash_daq-0.5.0-darkgrey) -![](https://img.shields.io/badge/dash_leaflet-1.0.15-darkgrey) +![](https://img.shields.io/badge/dash_deck-0.5.0-darkgrey) +![](https://img.shields.io/badge/dash_daq-0.0.1-darkgrey) +![](https://img.shields.io/badge/dash_html_components-2.0.0-darkgrey) +![](https://img.shields.io/badge/dash_resizable_panels-0.1.0-darkgrey) ![](https://img.shields.io/badge/dash_svg-0.0.12-darkgrey) -![](https://img.shields.io/badge/holoviews-1.18.3-darkgrey) -![](https://img.shields.io/badge/hvplot-0.9.2-darkgrey) +![](https://img.shields.io/badge/Flask-3.0.3-darkgrey) +![](https://img.shields.io/badge/Flask_Caching-2.3.0-darkgrey) ![](https://img.shields.io/badge/Flask_Compress-1.15-darkgrey) +![](https://img.shields.io/badge/Requests-2.32.3-darkgrey) ![](https://img.shields.io/badge/httpx-0.27.2-darkgrey) ![](https://img.shields.io/badge/pandas-2.2.1-darkgrey) +![](https://img.shields.io/badge/geopandas-0.14.2-darkgrey) ![](https://img.shields.io/badge/numpy-1.26.4-darkgrey) -![](https://img.shields.io/badge/plotly-5.18.0-darkgrey) ![](https://img.shields.io/badge/rasterio-1.3.9-darkgrey) ![](https://img.shields.io/badge/xarray-2024.1.1-darkgrey) +![](https://img.shields.io/badge/rioxarray-0.15.5-darkgrey) +![](https://img.shields.io/badge/Pillow-10.4.0-darkgrey) +![](https://img.shields.io/badge/pyproj-10.4.0-darkgrey) +![](https://img.shields.io/badge/PyYAML-6.0.1-darkgrey) +![](https://img.shields.io/badge/Shapely-2.0.6-darkgrey) ## Run GRIDCERF Locally diff --git a/dash_app/app.py b/dash_app/app.py index 1011f63..4445486 100644 --- a/dash_app/app.py +++ b/dash_app/app.py @@ -2,58 +2,35 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------------- -# GRIDCERF, data tool and app that runs visuals on Closed-Loop Geothermal Data +# GRIDCERF, U.S energy feasibility mapper and database explorer # ------------------------------------------------------------------------------------- -# https://dash.gallery/dash-deck-explorer/globe-view - -# THIS is what I want for a quick introduction to JavaScript -# https://github.com/Esri/quickstart-map-js - - -# https://blog.mapbox.com/visualizing-radar-data-with-vector-tiles-117bc5ee9a5a - # LIBRARIES -# standard libraries +## standard libraries import os import sys import yaml -# data manipulation -import pandas as pd -import numpy as np -import xarray as xr -from functools import reduce - -# web visualization and interactive libraries +## web visualization and interactive libraries from dash.dependencies import Input, Output, State from dash import Dash, html from dash import ctx from dash.exceptions import PreventUpdate from dash import dcc -import holoviews as hv -hv.extension('bokeh') -from holoviews.plotting.plotly.dash import to_dash # SOURCED SCRIPTS -from definitions import CONNECT_TO_LAMBDA, PORT, plotly_config, CMAP_BLACK, token +from definitions import CONNECT_TO_LAMBDA, PORT, COMPILED_DIR, OUTDIR if CONNECT_TO_LAMBDA: from msdlive_utils import get_bytes from io import BytesIO from src.reader import open_as_raster -from src.utilities import create_datashaded_scatterplot #, get_map_data -from src.imshow import plot_imshow_map -from src.datashader_mapbox import plot_ds_mapbox_map -from src.datashader_holoviews import plot_ds_holoviews_map -from src.leaflet_titiler import plot_leaflet_map -from src.mapbox_raster import plot_mapbox_map from src.deckgl import plot_deckgl_map -from layout import app, tech_pathways_df, src_meta, all_options, OUTDIR, COMPILED_DIR +from layout import app, tech_pathways_df, src_meta, all_options # ----------------------------------------------------------------------------- -# Define dash app plotting callbacks. +# Define dash app callbacks. # ----------------------------------------------------------------------------- @app.callback( @@ -154,8 +131,6 @@ def set_level2_value(available_options): def show_hide_element(feature, is_ccs, cooling, capacity_factor): - # eventually integrate this into the above callbacks - feature_show = {'display': 'block'} is_css_show = {'display': 'block'} cooling_show = {'display': 'block'} @@ -179,34 +154,29 @@ def show_hide_element(feature, is_ccs, cooling, capacity_factor): @app.callback( Output(component_id="map", component_property="children"), [ - Input(component_id="map-select", component_property="value"), - # Input(component_id="state-select", component_property="value"), Input(component_id="year-select", component_property="value"), - Input(component_id="ssp-select", component_property="value"), # Socioeconomic scenario (SSP) + Input(component_id="ssp-select", component_property="value"), Input(component_id="tech-select", component_property="value"), Input(component_id="subtech-select", component_property="value"), Input(component_id="feature-select", component_property="value"), Input(component_id="carbon-capture-select", component_property="value"), Input(component_id="cooling-type-select", component_property="value"), Input(component_id="capacity-factor-select", component_property="value"), - # e.g., communications perspective: Land Management/Native Habitats - # Input vs. Compiled (will always be compiled) + Input(component_id="layer-selector", component_property="value") ], ) -def map(maptool, #state, - year, ssp, - tech, subtech, feature, - is_ccs, coolingtype, capacity_factor): +def map(year, ssp, + tech, subtech, feature, + is_ccs, coolingtype, capacity_factor, selected_layers): # ----------------------------------------------------------------------------- # Creates and displays map by querying a "database" table of all pathways # to their filenames. # ----------------------------------------------------------------------------- - year = str(year) - print(" --------------------------------------------------------------- ") + year = str(year) print([ssp, year, tech, subtech, feature, is_ccs, coolingtype, capacity_factor]) query_df = tech_pathways_df.query("ui_ssp in @ssp and \ ui_year in @year and \ @@ -218,45 +188,15 @@ def map(maptool, #state, ui_capacity_factor in @capacity_factor") fpaths = query_df["fpath"].values - # print(len(fpaths)) # i = 0 # for fpath in fpaths: - # TIFPATH = os.path.join(COMPILED_DIR, fpath) # data_array, array, source_crs, df_coors_long, boundingbox, img = open_as_raster(TIFPATH=TIFPATH, is_reproject=False, is_convert_to_png=False) # i += 1 - if maptool == "Plotly-datashader, mapbox": - - fig_div = plot_ds_mapbox_map(COMPILED_DIR=COMPILED_DIR, fpaths=fpaths) - - if maptool == "Plotly-datashader, holoviews": - - fig = plot_ds_holoviews_map(COMPILED_DIR=COMPILED_DIR, fpaths=fpaths) - - components = to_dash(app, [fig]) # breaks here, reset_button=True - - fig_div = html.Div(id="energy-map3", - children=components.children - ) - print("HOLOVIEWS PLOTTED") - - if maptool == "Leaflet and TiTiler": - - fig_div = plot_leaflet_map(COMPILED_DIR=COMPILED_DIR, fpaths=fpaths) - - if maptool == "Mapbox": - - fig_div = plot_mapbox_map(COMPILED_DIR=COMPILED_DIR, fpaths=fpaths) - - if maptool == "Plotly-imshow": - - fig_div = plot_imshow_map(COMPILED_DIR=COMPILED_DIR, fpaths=fpaths) - - if maptool == "DeckGL": - - fig_div = plot_deckgl_map(COMPILED_DIR=COMPILED_DIR, fpaths=fpaths) + # DeckGL + fig_div = plot_deckgl_map(COMPILED_DIR=COMPILED_DIR, fpaths=fpaths, selected_layers=selected_layers) return fig_div @@ -293,8 +233,15 @@ def output_string(active_cell): layer_methodology = src_meta.loc[data_row, "layer_methodology"] citation = src_meta.loc[data_row, "source_data_citation"] data_link = src_meta.loc[data_row, "source_data_link"] - - # tag_id = "TAG ID: " + str(tag_id) return str(title), str(tag_id), str(src_type), str(desc), str(date_updated), str(data_accessed), str(layer_methodology), str(citation), str(data_link) +# ----------------------------------------------------------------------------- +# App runs here. Define configurations, proxies, etc. +# ----------------------------------------------------------------------------- + +if CONNECT_TO_LAMBDA: + print("Sending app to the get_wsgi_handler ... ") +else: + if __name__ == "__main__": + app.run_server(port=PORT, debug=True) \ No newline at end of file diff --git a/dash_app/assets/map.css b/dash_app/assets/map.css index 51b1628..d639322 100644 --- a/dash_app/assets/map.css +++ b/dash_app/assets/map.css @@ -49,3 +49,7 @@ /* 800*/ /* width: 1300px; */ } + +/* .loader-wrapper > div { + visibility: visible !important; +} */ \ No newline at end of file diff --git a/dash_app/definitions.py b/dash_app/definitions.py index 5125b68..82f8d58 100644 --- a/dash_app/definitions.py +++ b/dash_app/definitions.py @@ -2,66 +2,33 @@ # -*- coding: utf-8 -*- import os -import matplotlib.colors as mcolors -# client (browser) paths +# CLIENT (BROWSER) PATHS PORT = int(os.environ.get("PORT", 8060)) REQUESETS_PATHNAME_PREFIX = "/" -CONNECT_TO_LAMBDA = False +CONNECT_TO_LAMBDA = True +# FILE PATHS if CONNECT_TO_LAMBDA: - DATASET_ID = "1ffea-emt93" # MSD-LIVE added dataset id that goes to DEV - -REMINDER = "It's coors = (lat, lon) and ... LON = COLS = X ... LAT = ROWS = Y" - -CMAP_BLACK = mcolors.ListedColormap(['black']) -# CMAP_BLACK = mcolors.ListedColormap((0,0,0,0.8)) + # https://gridcerf.dev.msdlive.org/ -token = open("../../mapbox_token.py").read() # mapbox_api_token = os.getenv("MAPBOX_ACCESS_TOKEN") - -plotly_config = {'displaylogo': False, - 'modeBarButtonsToRemove': ['autoScale', 'resetScale'], # High-level: zoom, pan, select, zoomIn, zoomOut, autoScale, resetScale - 'toImageButtonOptions': { - 'format': 'png', # one of png, svg, jpeg, webp - 'filename': 'custom_image', - 'height': None, - 'width': None, - 'scale': 6 # Multiply title/legend/axis/canvas sizes by this factor - } - } + DATASET_ID = "1ffea-emt93" # MSD-LIVE added dataset id that goes to DEV + DATA_DIR = "" + LAMBDA_TASK_ROOT = os.getenv('LAMBDA_TASK_ROOT') + if LAMBDA_TASK_ROOT is None: + METADATA_DIR = './metadata' + else: + METADATA_DIR = os.path.join(LAMBDA_TASK_ROOT, "dash_app", "metadata") -# https://medium.com/plotly/introducing-dash-holoviews-6a05c088ebe5 -# https://examples.holoviz.org/gallery/nyc_taxi/nyc_taxi.html -# https://towardsdatascience.com/displaying-a-gridded-dataset-on-a-web-based-map-ad6bbe90247f -# https://holoviews.org/getting_started/Customization.html -# https://holoviews.org/reference/elements/matplotlib/Points.html -# https://stackoverflow.com/questions/57588857/matplotlib-listedcolormap-transparent-color -# https://datashader.org/getting_started/Pipeline.html -# https://holoviews.org/_modules/holoviews/operation/datashader.html -# https://discourse.holoviz.org/t/how-to-set-geoviews-map-extent-programmatically-in-panel-dashboard/1181 -# https://geog-312.gishub.org/book/geospatial/leafmap.html -# https://leafmap.org/get-started/ -# https://ai-incubator.pnnl.gov/chat/fy0s5mBnqiTK9YE6C4xB8YrmQv0YMnyLsjA7 -# https://www.google.com/search?q=holoviews+albers+conic&rlz=1C5CHFA_enUS1091US1091&oq=holoviews+albers+conic&gs_lcrp=EgZjaHJvbWUyBggAEEUYOTIHCAEQIRigATIHCAIQIRigATIHCAMQIRigATIHCAQQIRigAdIBCDQ0NTdqMGo3qAIAsAIA&sourceid=chrome&ie=UTF-8 -# https://geoviews.org/user_guide/Projections.html +else: + DATA_DIR = "../../data/msdlive-gridcerf" + METADATA_DIR = "./metadata" -# https://plotly.com/python/mapbox-layers/ -# learn Mapbox GL JS: https://docs.mapbox.com/mapbox-gl-js/guides/migrate/ -# layer building in holoviz: https://github.com/holoviz/holoviews/issues/3882 -# remove frame around plot: https://discourse.holoviz.org/t/remove-frame-around-plot/4023/3 -# LINKED SELECTIONS ** https://medium.com/plotly/introducing-dash-holoviews-6a05c088ebe5 -# foreal example: -# https://github.com/niowniow/foreal/blob/ea0cb314f3a408f7fd544528ce83e03ca2269b12/foreal/webportal/app.py#L48 -# https://github.com/niowniow/foreal/blob/ea0cb314f3a408f7fd544528ce83e03ca2269b12/foreal/webportal/app.py#L48 -# https://github.com/niowniow/foreal/blob/ea0cb314f3a408f7fd544528ce83e03ca2269b12/foreal/webportal/__init__.py -# https://github.com/niowniow/foreal/blob/ea0cb314f3a408f7fd544528ce83e03ca2269b12/foreal/webportal/shared.py +COMPILED_DIR = os.path.join(DATA_DIR, "gridcerf/compiled/compiled_technology_layers") +OUTDIR = "tmp" -# stocknews example: -# https://github.com/troyscribner/stocknews/tree/6565a854a3469c93b42f2a871301469aa6b9bd04 -# of interest: mapboxgl dash python -# click event: does dash python support mapboxgl -# +# REMINDER = "It's coors = (lat, lon) and ... LON = COLS = X ... LAT = ROWS = Y" diff --git a/dash_app/holoview_plot.html b/dash_app/holoview_plot.html deleted file mode 100644 index 211cedd..0000000 --- a/dash_app/holoview_plot.html +++ /dev/null @@ -1,66 +0,0 @@ - - - - - holoview_plot - - - - - - - - - - -
- - - - - \ No newline at end of file diff --git a/dash_app/holoview_plot2.html b/dash_app/holoview_plot2.html deleted file mode 100644 index 65e5f22..0000000 --- a/dash_app/holoview_plot2.html +++ /dev/null @@ -1,66 +0,0 @@ - - - - - holoview_plot2 - - - - - - - - - - -
- - - - - \ No newline at end of file diff --git a/dash_app/layout.py b/dash_app/layout.py index 8946d38..5175acc 100644 --- a/dash_app/layout.py +++ b/dash_app/layout.py @@ -5,76 +5,55 @@ # Define dash app layout. # ----------------------------------------------------------------------------- -# standard libraries +# LIBRARIES + +## standard libraries import os import sys -# data manipulation +## data manipulation import pandas as pd -# web app and interactive graphics libraries +## AWS libraries from flask import Flask from flask_compress import Compress + +## web framework import dash from dash import html, dcc -import dash_bootstrap_components as dbc # Adds bootstrap components for more web themes and templates -import dash_leaflet as dl +import dash_bootstrap_components as dbc from dash import dash_table -from dash_resizable_panels import PanelGroup, Panel, PanelResizeHandle +from flask_caching import Cache +cache = Cache(config={'CACHE_TYPE': 'SimpleCache'}) -# sourced scripts +# SOURCED SCRIPTS from src.utilities import recur_dictify -from definitions import CONNECT_TO_LAMBDA, REQUESETS_PATHNAME_PREFIX - -if CONNECT_TO_LAMBDA: - - DATA_DIR = "" - # MSD-LIVE added logic to support app running locally and in lambda: - LAMBDA_TASK_ROOT = os.getenv('LAMBDA_TASK_ROOT') - # If 'LAMBDA_TASK_ROOT' is not set, METADATA_DIR will be './metadata' - # If 'LAMBDA_TASK_ROOT' is set, METADATA_DIR will be the absolute path - METADATA_DIR = './metadata' if LAMBDA_TASK_ROOT is None else os.path.join(LAMBDA_TASK_ROOT, "dash_app", "metadata") - # client (browser) paths - -else: - - # file paths and global variables - DATA_DIR = "../../data/msdlive-gridcerf" - METADATA_DIR = os.path.join(DATA_DIR, "metadata") - +from definitions import CONNECT_TO_LAMBDA, REQUESETS_PATHNAME_PREFIX, METADATA_DIR, OUTDIR -COMPILED_DIR = os.path.join(DATA_DIR, "gridcerf/compiled/compiled_technology_layers") -OUTDIR = "tmp" - -# central and queryable -tech_pathways_df = pd.read_csv(os.path.join(METADATA_DIR, "msdlive_tech_paths.csv")) # tech_pathways_mapped.csv -src_meta = pd.read_csv(os.path.join(METADATA_DIR, "metadata_ab_edits.csv")) +# PATHS +tech_pathways_df = pd.read_csv(os.path.join(METADATA_DIR, "msdlive_tech_paths.csv")) +src_meta = pd.read_csv(os.path.join(METADATA_DIR, "metadata_ab_edits.csv")) ## sourced src_meta_df = src_meta[["plain_language_layer_name", "source_tag_id", "source_data_title"]] tech_pathways_df = tech_pathways_df.fillna('--') -tech_pathways_df['ui_year'] = tech_pathways_df['ui_year'].astype(str) +tech_pathways_df['ui_year'] = tech_pathways_df['ui_year'].astype(str) ## sourced + pathway = ["ui_tech", "ui_subtype", "ui_feature", "ui_is_ccs", "ui_cooling_type", "ui_capacity_factor"] tech_pathways_dict = recur_dictify(df=tech_pathways_df[pathway]) -all_options = tech_pathways_dict - +all_options = tech_pathways_dict ## sourced # ----------------------------------------------------------------------------- # Dash app layout begins here. # ----------------------------------------------------------------------------- -# MSD-LIVE TODO need to wrap in a function to support lambda -def create_app(): #-> Dash: +def create_app(): server = Flask(__name__) Compress(server) - if CONNECT_TO_LAMBDA: + app = dash.Dash(__name__, assets_folder="assets", - # Bootstrap stylesheets available on the web: https://dash-bootstrap-components.opensource.faculty.ai/docs/themes/ - # Link to a stylesheet served over a content delivery network (CDN) - # BOOTSTRAP links to the standard Bootstrap stylesheet external_stylesheets=[dbc.themes.BOOTSTRAP], - # url_base_pathname=URL_BASE_PATHNAME, # not needed requests_pathname_prefix=REQUESETS_PATHNAME_PREFIX, meta_tags=[ {"charset": "UTF-8"}, @@ -82,13 +61,9 @@ def create_app(): #-> Dash: {"name": "viewport", "content": "width=device-width, initial-scale=1"}, {"name": "description", "content": "Geospatial Raster Input Data for Capacity Expansion Regional Feasibility (GRIDCERF). A high-resolution energy mapper."} ], - - # MSD-LIVE compressing the response via flask-compress, can do both locally and for lambda - # MSD-LIVE add logic to set serve_locally to support lambda - # You must set serve_locally=False or the app will not work when deployed remotely - serve_locally = False if LAMBDA_TASK_ROOT is not None else True, + serve_locally = False if LAMBDA_TASK_ROOT is not None else True, # must be False for app deployment on AWS lambda server=server - ) + ) else: @@ -102,44 +77,11 @@ def create_app(): #-> Dash: {"name": "description", "content": "Geospatial Raster Input Data for Capacity Expansion Regional Feasibility (GRIDCERF). A high-resolution energy mapper."} ], ) - - app.index_string = ''' - - - - {%metas%} - {%css%} - - - - - - - - {%app_entry%} - {%config%} - {%scripts%} - {%renderer%} - - - ''' - # print("did it work?") - + cache.init_app(app.server) + app.title = "GRIDCERF | Geospatial Raster Input Data for Capacity Expansion Regional Feasibility" - - plotly_config = {'displaylogo': False, - 'modeBarButtonsToRemove': ['autoScale', 'resetScale'], # High-level: zoom, pan, select, zoomIn, zoomOut, autoScale, resetScale - 'toImageButtonOptions': { - 'format': 'png', # one of png, svg, jpeg, webp - 'filename': 'custom_image', - 'height': None, - 'width': None, - 'scale': 6 # Multiply title/legend/axis/canvas sizes by this factor - } - } - # ----------------------------------------------- # HTML components. # ----------------------------------------------- @@ -155,37 +97,32 @@ def create_app(): #-> Dash: "South Carolina", "South Dakota", "Tennessee", "Texas", "Utah", "Virginia", "Virgin Islands", "Vermont", "Washington", "Wisconsin", "West Virginia", "Wyoming"] - tabs = ["information-tab", "insights-tab", "layers-tab"] section_headers = ["Overview", "Authors", "Funding"] - title_text = """Geospatial Raster Input Data for Capacity Expansion Regional Feasibility (GRIDCERF)""" + title_text = """Geospatial Raster Input Data for Capacity Expansion Regional Feasibility (GRIDCERF) Version 2.0""" - description_text = """A high-resolution energy mapper for exploring the siting viability of renewable - and non-renewable power plants in the United States.""" + description_text = """A high-resolution energy mapper for exploring the siting suitability of renewable + and non-renewable power plants in the contiguous United States.""" - # overview_text = """The United States will likely need new utility-scale generation resources because rising - # electricity demand cannot be met by energy efficiency initiatives alone. - # Climate change, energy system transitions, and socioeconomic shifts are also driving - # electricity demand and power plant siting.""" - # overview_text_cont = """To explore where new power plants can be built, GRIDCERF geovisualizes technology-specific - # data comprised of 264 suitability layers across 56 power plant technologies. The data are fully - # compatible with integrated, multi-model approaches, so they can easily be re-ingested into - # geospatial modeling software.""" + overview_text = """The GRIDCERF database is a high-resolution product to evaluate siting suitability for renewable + and non-renewable power plants in the conterminous United States. GRIDCERF offers hundreds of + individual suitability layers for use with both renewable and non-renewable power plant + technology configurations in a harmonized format that can be easily ingested by + geospatially-enabled modeling software. + """ + + overview_text2 = """ GRIDCERF data can be directly used with the power plant siting model CERF + (Capacity Expansion Regional Feasibility) to site power plants at a 1km2 resolution.""" - overview_text = """The United States will likely need new utility-scale generation resources because rising - electricity demand cannot be met by energy efficiency initiatives alone. To explore - where new power plants can be built, GRIDCERF geovisualizes technology-specific - data comprised of 264 suitability layers across 56 power plant technologies. The data are fully - compatible with integrated, multi-model approaches, so they can easily be re-ingested into - geospatial modeling software.""" + overview_text3 = """Download GRIDCERF data from MSDLIVE.""" author_text = """GRIDCERF represents the extensive collection of data formatting, processing, and visualization - created by the IM3 Group.""" # at the Pacific Northwest National Laboratory.""" + created by the IM3 Group.""" funding_text = """This research was funded by the U.S. Department of Energy, Office of Science, as part of - research in MultiSector Dynamics, Earth and Environmental Systems Modeling Program.""" + research in MultiSector Dynamics, Earth and Environmental Systems Modeling Program.""" select_headers = ["Select a visualization tool", "Select a state", @@ -196,9 +133,9 @@ def create_app(): #-> Dash: "Select a Cooling Type", "Select a Shared Socioeconomic Pathway (SSP)", # Select a socioeconomic scenario "Select a feature", - "Select a Capacity Factor (CF)" - + "Select a Class" # Capacity Factor (CF) ] + select_ids = ["map-select", "state-select", "year-select", @@ -240,7 +177,7 @@ def metadata_text_value(group_id, text_id, value_id, text): ] ) - def metadata_card(): + def layer_metadata_card(): return html.Div( id="meta", @@ -271,37 +208,6 @@ def metadata_card(): def tabs_card(): - # information_tab = dcc.Tab(label="Information", - # id=tabs[0], - # value=tabs[0], - # selected_className="active-tab", - # children=[ - # # html.Hr(className="tab-hr"), - # html.Div(id='intro', - # children=[html.P(title_text, id='title', className="title-text"), - # html.P(description_text, id='description-text', className="page-text"), - # html.P(section_headers[0], id='header0', className="header-text"), - # html.Hr(className="hr"), - # html.P(overview_text, id='overview-text', className="page-text"), - # html.P(overview_text_cont, id='overview-text-cont', className="page-text"), - # html.P(section_headers[1], id='header1', className="header-text"), - # html.Hr(className="hr"), - # html.P(author_text, id='author-text', className="page-text"), - # html.P(section_headers[2], id='header2', className="header-text"), - # html.Hr(className="hr"), - # html.P(funding_text, id='funding-text', className="page-text"), - # # html.Label([html.P("Download the contributing", id="shorttext1"), - # # html.A('papers', href='https://gdr.openei.org/submissions/1473', id='hyperlink1'), - # # html.P("and", id="shorttext2"), - # # html.A('code', href='https://github.com/pnnl/GeoCLUSTER', id='hyperlink2'), - # # html.P(".", id="shorttext3"), - # # ], id='ab-note4') - # ]), - # # html.Button("Get Started", id="launch-btn", className="button"), - # ] - # ) - - insights_tab = dcc.Tab(label="Technology Suitability", id=tabs[1], value=tabs[1], @@ -310,22 +216,22 @@ def tabs_card(): html.Br(), - html.Div(id="map-select-container", - className="select-container", - children=[ - html.P(select_headers[0], id='select-header0', className="dropdown-header-text"), - dcc.Dropdown( - id="map-select", - className="dropdown-select", - options=["Plotly-imshow","Plotly-datashader, mapbox", "Plotly-datashader, holoviews", "Leaflet and TiTiler", "Mapbox", "DeckGL"], - value="DeckGL", # "Mapbox", # "Plotly-imshow", - clearable=False, - searchable=False, - multi=False - ), - - ] - ), + # html.Div(id="map-select-container", + # className="select-container", + # children=[ + # html.P(select_headers[0], id='select-header0', className="dropdown-header-text"), + # dcc.Dropdown( + # id="map-select", + # className="dropdown-select", + # options=["Plotly-imshow","Plotly-datashader, mapbox", "Plotly-datashader, holoviews", "Leaflet and TiTiler", "Mapbox", "DeckGL"], + # value="DeckGL", # "Mapbox", # "Plotly-imshow", + # clearable=False, + # searchable=False, + # multi=False + # ), + + # ] + # ), # html.P(select_headers[1], id='select-header1', className="dropdown-header-text"), # dcc.Dropdown( @@ -337,9 +243,6 @@ def tabs_card(): # searchable=False, # multi=False # ), - - # can also make year a slider from 2025 - 2100 with 5 year increments, - # OR make it an animation html.Div(id="tech-select-container", className="select-container", @@ -478,7 +381,7 @@ def tabs_card(): children=[ html.Br(), table_card(), - metadata_card() + layer_metadata_card() ] ) @@ -493,11 +396,11 @@ def tabs_card(): - def banner_card(): + def header_card(): """Builds the banner at the top of the page containing app name and logos. """ - return html.Header( # html.Header + return html.Header( id="banner", className="bannerbar", children=[ @@ -514,15 +417,6 @@ def banner_card(): ), ] ), - # html.A( - # href="https://www.pnnl.gov/", - # target="_blank", - # children=[ - # html.Img(id="lab-logo", className="logo", alt="lab logo", - # src=app.get_asset_url("icons/logos_icons/pnnl_abbreviated_logo.png")), - # ] - # ), - html.A( href="https://im3.pnnl.gov/", target="_blank", @@ -609,16 +503,37 @@ def footer_card(): def map(): return html.Div( - id="map", - className="map-column", children=[ - # dcc.Graph(id="energy-map", config=plotly_config), - # dl.Map(dl.TileLayer(), center=[56,10], zoom=6, style={'height': '50vh'}) - + dcc.Checklist( + id='layer-selector', + options=[ + {'label': 'Basemap Ocean', 'value': 'base-map-ocean'}, # AB: need to predfine the database + {'label': 'Basemap Land', 'value': 'base-map'}, + {'label': 'Feasibility Layer', 'value': 'feasibility-layer'}, + ], + value=["base-map-ocean", "base-map"], # Default selected layers + inline=True, + style={'display': 'none'} + ), + dcc.Loading( + id="loading", + type="circle", + # style={"backgroundColor": "transparent"}, + children=[ + html.Div( + id="map", + className="map-column", + children=[ + ] + ) + ] + ) ] - ) + ) + + def about(): return html.Div( @@ -632,10 +547,13 @@ def about(): html.P(section_headers[0], id='header0', className="header-text"), html.Hr(className="hr"), html.P(overview_text, id='overview-text', className="page-text"), + html.P(overview_text2, id='overview-text2', className="page-text"), + html.A(overview_text3, href="https://doi.org/10.57931/2281697", target="_blank", id='overview-text3', className="page-text"), + html.Br(), + html.Br(), + # html.P(overview_text_cont, id='overview-text-cont', className="page-text"), - html.P(section_headers[1], id='header1', className="header-text"), - html.Hr(className="hr"), - html.P(author_text, id='author-text', className="page-text"), + # html.P(section_headers[1], id='header1', className="header-text"), html.P(section_headers[2], id='header2', className="header-text"), html.Hr(className="hr"), html.P(funding_text, id='funding-text', className="page-text"), @@ -646,17 +564,6 @@ def about(): # html.P(".", id="shorttext3"), # ], id='ab-note4') ]), - - - # html.Div( - # [ - # html.H2("Sidebar", className="display-4"), - # html.Hr(), - # html.P( - # "A simple sidebar layout with navigation links", className="lead" - # ), - # ]) - ] ) @@ -678,44 +585,6 @@ def page_card(): className="page", children=[ - # html.Div([ - # PanelGroup( - # id='panel-group', - # children=[ - # Panel( - # id='panel-1', - # children=[ - # about(), - # ], - # ), - # PanelResizeHandle( - # html.Div( - # style={"backgroundColor": "grey", "height": "100%", "width": "5px"}) - # ), - # Panel( - # id='panel-2', - # children=[ - # map() - # ], - # # style={"backgroundColor": "black", "color": "white"} - # ), - # PanelResizeHandle( - # html.Div( - # style={"backgroundColor": "grey", "height": "100%", "width": "5px"}) - # ), - # Panel( - # id='panel-3', - # children=[ - # nav() - # ], - # # style={"backgroundColor": "black", "color": "white"} - # ) - # ], direction='horizontal' - # ) - # ], style={"height": "100vh"}) - - - about(), map(), nav(), @@ -727,7 +596,7 @@ def page_card(): id="app-container", children=[ # dcc.Store(id='results'), - banner_card(), + header_card(), page_card(), footer_card(), ], diff --git a/dash_app/other.py b/dash_app/other.py deleted file mode 100644 index c9b085f..0000000 --- a/dash_app/other.py +++ /dev/null @@ -1,15 +0,0 @@ -# import time -# import pydoc -# from bokeh.sampledata.penguins import data as dftest -# import hvplot.xarray -# import hvplot.pandas -# import holoviews as hv -# from holoviews.plotting.plotly.dash import to_dash -# from holoviews.operation.datashader import datashade -# hv.extension("plotly") -# from plotly.data import carshare -# from plotly.colors import sequential - - -# import dash_daq as daq # Adds more data acquisition (DAQ) and controls to dash callbacks -# from dash_svg import Svg, G, Path, Circle # Scalable Vector Graphics (SVG) maker \ No newline at end of file diff --git a/dash_app/run.py b/dash_app/run.py deleted file mode 100644 index 64c2160..0000000 --- a/dash_app/run.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -from app import app -from definitions import PORT - -# ----------------------------------------------------------------------------- -# App runs here. Define configurations, proxies, etc. -# ----------------------------------------------------------------------------- - -# server = app.server -# from app import server as application # in the wsgi.py file -- this targets the Flask server of Dash app - -if __name__ == "__main__": - - # app = create_app() - app.run_server(port=PORT, debug=True) - # app.run_server(debug=True, use_reloader=False) # Turn off reloader if inside Jupyter diff --git a/dash_app/src/datashader_holoviews.py b/dash_app/src/datashader_holoviews.py deleted file mode 100644 index dab29b6..0000000 --- a/dash_app/src/datashader_holoviews.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# standard -import os - -# visualization and data manipulation -import holoviews as hv -from holoviews import opts -from holoviews.operation.datashader import datashade -from holoviews.element.tiles import EsriImagery, CartoDark, CartoLight, EsriStreet, EsriNatGeo, EsriUSATopo, EsriTerrain, OSM, OpenTopoMap, EsriReference - -# sourced scripts -from src.reader import open_as_raster -from definitions import plotly_config, CMAP_BLACK - - -def plot_ds_holoviews_map(COMPILED_DIR, fpaths): - - - TIFPATH = os.path.join(COMPILED_DIR, fpaths[0]) - data_df, array, source_crs, geo_crs, df_coors_long, boundingbox, img = open_as_raster(TIFPATH=TIFPATH, is_reproject=True, is_convert_to_png=False) - - # to use RAPIDS ... need to install drivers, but have no GPUs on Mac - - # ylat = np.arange(0, array.shape[0], 1).tolist() - # xlon = np.arange(0, array.shape[1], 1).tolist() - - # data_xr = xr.DataArray(array, - # coords={'y': ylat,'x': xlon}, - # dims=["y", "x"]) - # df = data_xr.to_dataframe(name='value').reset_index() - # print(df) - - # this works - # fig = df_coors_long.hvplot.scatter(x='LongitudeProj', y='LatitudeProj', by='IsFeasible') - # fig = df_coors_long.hvplot(x="LongitudeProj", y="LatitudeProj", kind='scatter', rasterize=True) - plot_width = int(5460/1.32) - plot_height = int((3229-250)/1.32) - # print(int(5460/1.32), int((3229-250)/1.32)) - - # map_tiles = gv.tile_sources.OSM() # gv not defined does not work - # map_tiles = EsriImagery().opts(alpha=0.5, width=plot_width, height=plot_height, bgcolor='black') - # map_tiles = hv.element.tiles.EsriImagery().opts(alpha=0.5, width=plot_width, height=plot_height, show_grid=False) - # map_tiles = CartoDark() - map_tiles = EsriStreet().opts(alpha=0.5, bgcolor='black', width=plot_width, height=plot_height) - # map_tiles = hv.Tiles('https://tile.openstreetmap.org/{Z}/{X}/{Y}.png', name="OSM") #.opts(width=plot_width, height=plot_height) - # all of the OSM maps here: https://xyzservices.readthedocs.io/en/stable/gallery.html - # map_tiles = hv.Tiles('https://tiles.stadiamaps.com/tiles/stamen_toner_background/{z}/{x}/{y}{r}.{ext}', name="OSM") - - df_coors_long["easting"], df_coors_long["northing"] = hv.Tiles.lon_lat_to_easting_northing( - df_coors_long["LongitudeProj"], df_coors_long["LatitudeProj"] - ) - # points = hv.Points(df_coors_long, ['LongitudeProj', 'LatitudeProj']) - df_coors_long["alpha"] = 1 # get an error if put 100 - df_coors_long["color"] = "black" - # df_coors_long["marker"] = "circle" - # df_coors_long["size"] = 1 - - # fig.opts.defaults(opts.Points(size=1, line_color='black')) - - # SHOW - # STREAMS****: https://holoviews.org/gallery/apps/bokeh/nytaxi_hover.html - points = hv.Points(df_coors_long, ['easting', 'northing'], vdims=['alpha', 'color']).opts( - cmap=CMAP_BLACK, alpha="alpha", - color="color", - # color='k' - # marker='marker', #size='size' - ) - # markers = points.opts( - # opts.Points( - # color='color', - # size='size', - # alpha='alpha', - # # tools=['hover'], - # cmap=CMAP_BLACK, # Or any color map - # size_index='size' - # ) - # ) - # https://holoviews.org/_modules/holoviews/operation/datashader.html - feasibility = datashade(points, cmap=CMAP_BLACK, - # color_key="#000000", - alpha=155, - min_alpha=255, - width=plot_width, height=plot_height) - fig = map_tiles * feasibility * points - - # hv.extension('bokeh', logo=False) - - - print(fig) - print(map_tiles) - - fig.opts(xaxis=None,yaxis=None, title="") - setattr(fig, 'plot_width', 1700) - setattr(fig, 'plot_height', 900) - - print(opts) - # plot_width = int(750) - # plot_height = int(plot_width*4) - opts.defaults( - opts.Points(size=1, color='blue'), - # opts.Overlay(width=plot_width, height=plot_height, xaxis=None, yaxis=None), - # opts.RGB(width=plot_width, height=plot_height), - # opts.Histogram(responsive=True, min_height=250) - ) - - - - # fig.update_layout(margin={"r":0,"t":0,"l":0,"b":0}) - - # Display the plot - # hv.save(fig, 'holoview_plot.html', fmt='html') - - # plot = df_coors_long.hvplot( - # x='easting', - # y='northing', - # kind='scatter', - # rasterize=True, - # cmap=cc.fire, - # cnorm='eq_hist', - # width=length, - # height=length*2 - # ) - # fig2 = map_tiles * plot - # hv.save(fig2, 'holoview_plot2.html', fmt='html') - - # scatter = create_datashaded_scatterplot(df) - - return fig diff --git a/dash_app/src/datashader_mapbox.py b/dash_app/src/datashader_mapbox.py deleted file mode 100644 index 3edd404..0000000 --- a/dash_app/src/datashader_mapbox.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# standard -import os - -# visualization and data manipulation -import numpy as np -import plotly.express as px -from dash import html, dcc -import datashader as ds -import holoviews as hv -import datashader.transfer_functions as tf - -# sourced scripts -from src.reader import open_as_raster -from definitions import plotly_config, CMAP_BLACK - - -def plot_ds_mapbox_map(COMPILED_DIR, fpaths): - - TIFPATH = os.path.join(COMPILED_DIR, fpaths[0]) - data_df, array, source_crs, geo_crs, df_coors_long, boundingbox, img = open_as_raster(TIFPATH=TIFPATH, is_reproject=True, is_convert_to_png=False) - - df_coors_long["easting"], df_coors_long["northing"] = hv.Tiles.lon_lat_to_easting_northing( - df_coors_long["LongitudeProj"], df_coors_long["LatitudeProj"] - ) - - # points = hv.Points(df_coors_long, ['easting', 'northing']) - # feasibility = datashade(points, cmap=cc.fire, width=plot_width, height=plot_height) - # fig = map_tiles * feasibility - - # cvs = ds.Canvas(plot_width=int(6858/1.5), plot_height=int(3033/1.5)) # 3033, 6858 - # 3229, 5460 - - # just feed the new data - - # bigger over all number, smaller fraction number, more holes - cvs = ds.Canvas(plot_width=int(5460/1.32), plot_height=int((3229-250)/1.32)) - agg = cvs.points(df_coors_long, x='LongitudeProj', y='LatitudeProj') - coords_lat, coords_lon = agg.coords['LatitudeProj'].values, agg.coords['LongitudeProj'].values - - print(int(5460/1.32), int((3229-250)/1.32)) # wow I was able to guess the length(LongitudeProj) and the length(LatitudeProj) is - # what the plot_width and plot_heights should be respectively! - print(agg) - # WALKTHROUGH: https://towardsdatascience.com/big-data-visualization-using-datashader-in-python-c3fd00b9b6fc - - # x_range: (-124.4600091773715, -67.0847028164778) - # y_range: (26.648993778952498, 48.99927661906988) - # Corners of the image, which need to be passed to mapbox (xarray) - coordinates = [[coords_lon[0], coords_lat[0]], - [coords_lon[-1], coords_lat[0]], - [coords_lon[-1], coords_lat[-1]], - [coords_lon[0], coords_lat[-1]]] - - print(coordinates) - - # print(df_coors_long[:1]) - - - img = tf.shade(agg, cmap=CMAP_BLACK)[::-1].to_pil() #gray, kbc (blue)_ cc.kg - fig = px.scatter_mapbox(df_coors_long, lat='LatitudeProj', lon='LongitudeProj', - color_discrete_sequence=["#48f542"], - # size="IsFeasible", - # size=1, - zoom=3) - - - fig.update_layout(mapbox_style="open-street-map", # "carto-darkmatter", - mapbox_layers = [ - { - "sourcetype": "image", - "source": img, - "coordinates": coordinates - }] - ) # could separate basemap here - - # https://plotly.com/python/datashader/ - - - - # https://developer.nvidia.com/blog/making-a-plotly-dash-census-viz-powered-by-rapids/ - - # 'data': [{ - # 'type': 'scattermapbox', - # 'lat': lat, 'lon': lon, - # }], - # 'layout': { - # 'mapbox': { - # … - # 'layers': [{ - # "sourcetype": "image", - # "source": datashader_output_img, - # }], - # } - - fig_div = html.Div(id="plotly-map", - children=[dcc.Graph(id="energy-map", figure=fig, config=plotly_config)] - ) - - print("DATASHADER + MAPBOX PLOTTED") - - return fig_div \ No newline at end of file diff --git a/dash_app/src/deckgl.py b/dash_app/src/deckgl.py index 8360883..6065a70 100644 --- a/dash_app/src/deckgl.py +++ b/dash_app/src/deckgl.py @@ -8,269 +8,152 @@ and column layers. The data used contains global plant database and can be found here: https://github.com/ajduberstein/geo_datasets -Notice that here, we are explicitly convert the r.to_json() into a python dictionary. -This is needed because the data contains NaN, which can't be parsed by the underlying -JavaScript JSON parser, but it can be parsed by Python's JSON engine. +WebGL for Performance +WebGL is typically faster for rendering large datasets. Make sure you are leveraging this +by using efficient layer types, such as ScatterplotLayer or HexagonLayer, which are +optimized for large datasets. +Ensure your data is in an efficient format (like Parquet or GeoJSON) for faster loading +and processing. Preprocess your data to reduce its size, if possible """ +# standard libraries import os import json +# visualization, data manipulation, and framework libraries import dash +from dash import html import dash_deck -import dash_html_components as html import pydeck -import pandas as pd +import numpy as np -# sourced scripts +# SOURCED SCRIPTS from src.reader import open_as_raster -from definitions import token as mapbox_api_token - - -COLOR_BREWER_BLUE_SCALE = [ - [240, 249, 232], - [204, 235, 197], - [168, 221, 181], - [123, 204, 196], - [67, 162, 202], - [8, 104, 172], -] +from layout import cache +from definitions import OUTDIR + +@cache.memoize(timeout=7200) # Cache for 2 hours +def load_large_data(layer_name, COMPILED_DIR, fpaths): + + if layer_name == "base-map-ocean": + + OCEANS = json.load(open('data/ne_50m_ocean.geojson', 'r', encoding='utf-8')) + + return pydeck.Layer( + "GeoJsonLayer", + id="base-map-ocean", + data=OCEANS, + stroked=False, + filled=True, + pickable=False, # False to remove tooltip over oceans + auto_highlight=True, + get_line_color=[60, 60, 60], # [134, 181, 209], [22, 36, 105], + get_fill_color=[0, 31, 72], + opacity=0.5, + ) + + if layer_name == "base-map": + + LAND = "https://d2ad6b4ur7yvpq.cloudfront.net/naturalearth-3.3.0/ne_50m_admin_0_scale_rank.geojson" + + return pydeck.Layer( + "GeoJsonLayer", + id="base-map", + data=LAND, + stroked=False, + filled=True, + get_line_color=[60, 60, 60], + get_fill_color=[66, 133, 55], # green [32, 145, 62], [200, 200, 200], [160, 160, 160] + ) + + if layer_name == "feasibility-layer": -def plot_deckgl_map(COMPILED_DIR, fpaths): + TIFPATH = os.path.join(COMPILED_DIR, fpaths[0]) + data_df, array, source_crs, geo_crs, df_coors_long, boundingbox, img = open_as_raster(TIFPATH=TIFPATH, is_reproject=True, is_convert_to_png=False) - COUNTRIES = "https://d2ad6b4ur7yvpq.cloudfront.net/naturalearth-3.3.0/ne_50m_admin_0_scale_rank.geojson" - # POWER_PLANTS = "https://raw.githubusercontent.com/ajduberstein/geo_datasets/master/global_power_plant_database.csv" - OCEANS = json.load(open('data/ne_50m_ocean.geojson', 'r', encoding='utf-8')) - - # df = pd.read_csv(POWER_PLANTS) - - # easy to create a left and right orientation to rotate as needed, this can be loose but will be great - - # def is_green(fuel_type): - # if fuel_type.lower() in ( - # "nuclear", - # "water", - # "wind", - # "hydro", - # "biomass", - # "solar", - # "geothermal", - # ): - # return [10, 230, 120] - # return [230, 158, 10] - - - # df["color"] = df["primary_fuel"].apply(is_green) - - # view_state = pydeck.ViewState(latitude=51.47, longitude=0.45, zoom=2) - view_state = pydeck.ViewState(latitude=39.8283, - longitude=-98.5795, + icon_data = { + "url": "/assets/icons/map_icons/blue_square.svg", # svg repo + "width": 242, + "height": 242, + "anchorY": 121, # set to 0 if want to position it at the bottom center of the icon + "anchorX": 121, # center of X + } + + df_coors_long["icon_data"] = np.where(df_coors_long.index >= 0, icon_data, None) + df_coors_long["Angle"] = np.where(df_coors_long["LongitudeProj"] <= -121, 15, + np.where(df_coors_long["LongitudeProj"].between(-121, -118), 13, + np.where(df_coors_long["LongitudeProj"].between(-118, -112), 10, + np.where(df_coors_long["LongitudeProj"].between(-112, -107), 8, + np.where(df_coors_long["LongitudeProj"].between(-107, -103), 5, + np.where(df_coors_long["LongitudeProj"].between(-103, -100), 2, + np.where(df_coors_long["LongitudeProj"].between(-93, -91), -2, + np.where(df_coors_long["LongitudeProj"].between(-91, -82), -5, + np.where(df_coors_long["LongitudeProj"].between(-82, -76), -10, + np.where(df_coors_long["LongitudeProj"] > -76, -15, 0)))))))))) + + return pydeck.Layer( + id="feasibility-layer", + type="IconLayer", + data=df_coors_long, + get_icon="icon_data", + get_size=1150, # or 120 if you want best coverage + size_units="meters", + # size_scale=1, + # iconAtlas: 'path/to/icon-atlas.png', ? + # icon_mapping=icon_mapping, + width_scale=20, + get_width=1000, + radius=500, + get_radius=500, + get_angle="Angle", + get_pixel_offset=[0,1], + get_position=["LongitudeProj", "LatitudeProj"], + pickable=True, + auto_highlight=True, + stroked=True, + transitions={ + # transition with a duration of 3000ms + 'get_position': 3000, # Transition duration in milliseconds + 'get_size': 3000, + # 'radius': 1000 + # getRadius: { + # duration: 3000, + # easing: d3.easeBackInOut, + # }, + }, + # update_triggers={ + # # 'get_position': ['year'], + # 'get_icon': ['IsFeasible'] + # } + # rgb(60, 220, 255) + ) + + +def plot_deckgl_map(COMPILED_DIR, fpaths, selected_layers): + + view_state = pydeck.ViewState(latitude=39.8283, longitude=-98.5795, # center U.S. zoom=2, # pitch=30, # ? # bearing=0 # ? ) + view = pydeck.View(type="_GlobeView", controller=True, width="100%", height="100%") # width=1000, height=700 + deck_layers = [] - TIFPATH = os.path.join(COMPILED_DIR, fpaths[0]) - data_df, array, source_crs, geo_crs, df_coors_long, boundingbox, img = open_as_raster(TIFPATH=TIFPATH, is_reproject=True, is_convert_to_png=False) - - icon_data = { - # Icon from Wikimedia, used the Creative Commons Attribution-Share Alike 3.0 - # Unported, 2.5 Generic, 2.0 Generic and 1.0 Generic licenses - "url": "/assets/icons/map_icons/blue_square.svg", - "width": 242, - "height": 242, - "anchorY": 121, # set to 0 if wnat to position it at the bottom center of the icon - "anchorX": 121, # center of X - } - - # icon_mapping = { - # "myIcon": { - # # x: 0, - # # y: 0, - # # width: 128, - # # height: 128, - # "anchor": [0.5, 0.5], #// Position the anchor at the bottom center of the icon - # # offset: [0, -10] // Offset the icon by 10 pixels upwards - # } - # } - - df_coors_long["icon_data"] = None - for i in df_coors_long.index: - df_coors_long["icon_data"][i] = icon_data - - layers = [] - # Set height and width variables - # view = pydeck.View(type="_GlobeView", controller=True, width=1000, height=700) - view = pydeck.View(type="_GlobeView", controller=True, width="100%", height="100%") - - # print(df_coors_long.columns) - - df_coors_long["Angle"] = 0 - for i in df_coors_long.index: - - # left - if df_coors_long["LongitudeProj"][i] <= -121: - df_coors_long["Angle"][i] = 15 - if df_coors_long["LongitudeProj"][i] > -121 and df_coors_long["LongitudeProj"][i] <= -118: - df_coors_long["Angle"][i] = 13 - if df_coors_long["LongitudeProj"][i] > -118 and df_coors_long["LongitudeProj"][i] <= -112: - df_coors_long["Angle"][i] = 10 - if df_coors_long["LongitudeProj"][i] > -112 and df_coors_long["LongitudeProj"][i] <= -107: - df_coors_long["Angle"][i] = 8 - if df_coors_long["LongitudeProj"][i] > -107 and df_coors_long["LongitudeProj"][i] <= -103: - df_coors_long["Angle"][i] = 5 - if df_coors_long["LongitudeProj"][i] > -103 and df_coors_long["LongitudeProj"][i] <= -100: - df_coors_long["Angle"][i] = 2 - - # right - if df_coors_long["LongitudeProj"][i] > -93 and df_coors_long["LongitudeProj"][i] <= -91: - df_coors_long["Angle"][i] = -2 - if df_coors_long["LongitudeProj"][i] > -91 and df_coors_long["LongitudeProj"][i] <= -82: - df_coors_long["Angle"][i] = -5 - if df_coors_long["LongitudeProj"][i] > -82 and df_coors_long["LongitudeProj"][i] <= -76: - df_coors_long["Angle"][i] = -10 - if df_coors_long["LongitudeProj"][i] > -76: - df_coors_long["Angle"][i] = -15 - - layers = [ - pydeck.Layer( - "GeoJsonLayer", - id="base-map-ocean", - data=OCEANS, - stroked=False, - filled=True, - pickable=False, #Key difference in this layer as we don't want tooltips when hovering over the ocean - auto_highlight=True, - get_line_color=[60, 60, 60], - # get_fill_color=[134, 181, 209], - # get_fill_color=[22, 36, 105], - get_fill_color=[0, 31, 72], - opacity=0.5, - ), - pydeck.Layer( - "GeoJsonLayer", - id="base-map", - data=COUNTRIES, - stroked=False, - filled=True, - get_line_color=[60, 60, 60], - get_fill_color=[66, 133, 55], # green - # get_fill_color=[32, 145, 62], - # get_fill_color=[200, 200, 200], - # get_fill_color=[160, 160, 160] - ), - pydeck.Layer( - type="IconLayer", - data=df_coors_long, - get_icon="icon_data", - get_size=1150, # or 120 if you want best coverage - size_units="meters", #need to get rid of border - # size_scale=1, - # iconAtlas: 'path/to/icon-atlas.png', ? - # icon_mapping=icon_mapping, - width_scale=20, - get_width=1000, - radius=500, - get_radius=500, - get_angle="Angle", - # (The rotating angle of each object, in degrees, default is 0) - get_pixel_offset=[0,1], - # get_width=1000, - get_position=["LongitudeProj", "LatitudeProj"], - pickable=True, - auto_highlight=True, - stroked=True, - transitions={ - # transition with a duration of 3000ms - 'get_position': 3000, # Transition duration in milliseconds - 'get_size': 3000, - # 'radius': 1000 - # getRadius: { - # duration: 3000, - # easing: d3.easeBackInOut, - # }, - }, - # update_triggers={ - # # 'get_position': ['year'], - # 'get_icon': ['IsFeasible'] - # } - # rgb(60, 220, 255) - ), - # pydeck.Layer( - # does't work - # type="HeatmapLayer", - # data=df_coors_long, - # get_position=["LongitudeProj", "LatitudeProj"], - # aggregation=pydeck.types.String("MEAN"), - # pickable=True, - # get_weight="IsFeasible", - # threshold=1, - # color_range=COLOR_BREWER_BLUE_SCALE, - # opacity=0.9 - # ), - # pydeck.Layer( - # type="GridCellLayer", # ColumnLayer, HexagonLayer, ScatterplotLayer - # id="feasibility-raster", - # # offset Disk offset from the position, relative to the radius. By default, the disk is centered at each position. - # # angle Disk rotation, counter-clockwise in degrees. - # # vertices - # data=df_coors_long, - # get_elevation="IsFeasible", - # elevation_scale=100, - # get_position=["LongitudeProj", "LatitudeProj"], - # width_scale=20, - # get_width=1000, # Set the width of the column - # pickable=True, - # stroked=False, # Whether to draw an outline around the disks, Only applies if extruded: false. - # auto_highlight=True, - # # wireframe=True, - # extruded=False, - # radius=500, - # get_radius=500, # Radius is given in meters - # get_fill_color=[180, 0, 200, 140], # Set an RGBA value for fill - # # get_fill_color="color", - # opacity=0.5, - # ), - ] - - """ - - 3) Try an animation ! - A) Need to have a df that has all years in it (need to experiment with this pipeline) - - data = pd.DataFrame({ - 'year': years, - 'latitude': latitudes, - 'longitude': longitudes - }) - - 4) Figure out layer controller and other controller panels - 5) Verify offset and accuracy! - - - - """ - - """ The GridCellLayer can render a grid-based heatmap. - It is a variation of the ColumnLayer. It takes - the constant width / height of all cells and bottom-left - coordinate of each cell. This is the primitive layer - rendered by CPUGridLayer after aggregation. - Unlike the CPUGridLayer, it renders one column for - each data object.""" + for layer in selected_layers: + layer = load_large_data(layer, COMPILED_DIR, fpaths) # Load data, cached if previously loaded + deck_layers.append(layer) r = pydeck.Deck( views=[view], initial_view_state=view_state, - layers=layers, - map_provider="mapbox", + layers=deck_layers, + # map_provider="mapbox", # removed + parameters={"cull": True}, # map_style=pydeck.map_styles.SATELLITE, # map_style="mapbox://styles/mapbox/light-v9", # Note that this must be set for the globe to be opaque - parameters={"cull": True}, # tooltip={"text": "{tags}"} - # does NOT EXIST: # animation_config={ # 'duration': 2000, # Duration of the animation in milliseconds @@ -279,7 +162,8 @@ def plot_deckgl_map(COMPILED_DIR, fpaths): # } ) - # r.to_html("icon_layer.html") + # TODO: save/download the map feature can be added this way + # r.to_html(os.path.join(OUTDIR, "icon_layer.html")) ####### # Animate over the years @@ -292,35 +176,21 @@ def plot_deckgl_map(COMPILED_DIR, fpaths): # layer, # data=year_data # ) + # then connect to mapgl - # how does it connect to the below?? + """ + Notice that here, we are explicitly convert the r.to_json() into a python dictionary. + This is needed because the data contains NaN, which can't be parsed by the underlying + JavaScript JSON parser, but it can be parsed by Python's JSON engine. + """ mapgl = html.Div( dash_deck.DeckGL( json.loads(r.to_json()), id="deck-gl", style={"background-color": "black"}, - tooltip={"text": "{IsFeasible} feasible, {LatitudeProj}, {LongitudeProj}"}, + # tooltip={"text": "{IsFeasible} feasible, {LatitudeProj}, {LongitudeProj}"}, ) ) - fig_div = html.Div(id="plotly-map", - children=[mapgl] - ) # does not work - return mapgl - - # app = dash.Dash(__name__) - - # app.layout = html.Div( - # dash_deck.DeckGL( - # json.loads(r.to_json()), - # id="deck-gl", - # style={"background-color": "black"}, - # tooltip={"text": "{name}, {primary_fuel} plant, {country}"}, - # ) - # ) - - - # if __name__ == "__main__": - # app.run_server(debug=True) \ No newline at end of file diff --git a/dash_app/src/dir2yaml.py b/dash_app/src/dir2yaml.py index ea9e163..f9062c3 100644 --- a/dash_app/src/dir2yaml.py +++ b/dash_app/src/dir2yaml.py @@ -5,8 +5,6 @@ # import sys import yaml -# find . -name ".DS_Store" -delete - def dir_to_dict(path): directory = {} diff --git a/dash_app/src/imshow.py b/dash_app/src/imshow.py deleted file mode 100644 index 7a58f3c..0000000 --- a/dash_app/src/imshow.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# standard -import os - -# visualization and data manipulation -import numpy as np -import plotly.express as px -from dash import html, dcc - -# sourced scripts -from src.reader import open_as_raster -from definitions import plotly_config - -def plot_imshow_map(COMPILED_DIR, fpaths): - - TIFPATH = os.path.join(COMPILED_DIR, fpaths[0]) - data_df, array, source_crs, df_coors_long, boundingbox, boundingbox_proj, img = open_as_raster(TIFPATH=TIFPATH, is_reproject=False, is_convert_to_png=False) - - # TODO - # 1) get the bounding box of the data - # 2) set that to basemap - # 3) extract the basemap(s) - # 3) place that first into px.imshow() and then add array trace with fig.add_trace() - # 4) change X and Y hovers into lon and lat - # 5) hover also tells you the geographical location - # 6) and then a mini-view can show a leaflet zoom in to that area - - # US vector and raster tiles (and more) here: https://data.maptiler.com/downloads/north-america/us/ - - R_mask = np.where(np.isnan(array), 176, array) - G_mask = np.where(np.isnan(array), 233, array) - B_mask = np.where(np.isnan(array), 235, array) - - R_mask = np.where(array==0, 0, R_mask) - G_mask = np.where(array==0, 0, G_mask) - B_mask = np.where(array==0, 0, B_mask) - - rgb_stack = np.stack((R_mask, G_mask, B_mask), axis=-1) - fig = px.imshow(rgb_stack) - - fig.update_layout(margin=dict(l=0, r=0, t=0, b=0)) - fig.update_xaxes(showticklabels=False) - fig.update_yaxes(showticklabels=False) - fig.update(layout_coloraxis_showscale=False) - # fig.update_traces(showlegend=False) - # fig.update_traces(marker_showscale=False) - # fig.write_html(os.path.join(OUTDIR, "map-layer.html")) - print("IMSHOW PLOTTED") - print("\n\n") - - fig_div = html.Div(id="plotly-map", - children=[dcc.Graph(id="energy-map", figure=fig, config=plotly_config)] - ) - - return fig_div diff --git a/dash_app/src/layer_maker.py b/dash_app/src/layer_maker.py deleted file mode 100644 index dcce583..0000000 --- a/dash_app/src/layer_maker.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import pandas as pd -import numpy as np - -def sum_rasters(raster_list) -> np.ndarray: - """Sum all rasters in the input list and return a reclassified array having - values of 0 == suitable and 1 == unsuitable. - - :param raster_list: List of full path with file name and extensions to the input raster files - :type raster_list: list - - :return: A 2D array of 0, 1 suitablity designation - - """ - - for idx, raster in enumerate(raster_list): - - if idx == 0: - final_array = raster_to_array(raster).astype(np.float32) - - else: - - this_array = raster_to_array(raster) - final_array += this_array - - # reclassify array back to 0 == suitable, 1 == unsuitable - return np.where(final_array == 0, 0, 1) - - -def raster_to_array(raster_file) -> np.ndarray: - """Read in a raster file and return a 2D NumPy array. - - :param raster_file: Full path with file name and extension to the input raster file - :type raster_file: str - - :return: A 2D array - - """ - - with rasterio.open(raster_file) as src: - return src.read(1) - - -def plot_raster(arr, title): - - fig, ax = plt.subplots(figsize=(10, 5)) - - # Define the colors you want - cmap = ListedColormap(["grey", "navy"]) - - # Define a normalization from values -> colors - norm = colors.BoundaryNorm([0, 1], 2) - - raster_plot = ax.imshow(arr, cmap=cmap) - - # Add a legend for labels - legend_labels = {"grey": "0", "navy": "1"} - - patches = [Patch(color=color, label=label) - for color, label in legend_labels.items()] - - ax.legend(handles=patches, - bbox_to_anchor=(1.15, 1), - facecolor="white") - - ax.set_title(title) - - ax.set_axis_off() - - return ax - - - \ No newline at end of file diff --git a/dash_app/src/leaflet_titiler.py b/dash_app/src/leaflet_titiler.py deleted file mode 100644 index 7744ba3..0000000 --- a/dash_app/src/leaflet_titiler.py +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# standard -import os - -# visualization and data manipulation -from dash import html, dcc -import dash_leaflet as dl -import dash_leaflet.express as dlx - -# sourced scripts -from src.reader import open_as_raster -from definitions import plotly_config, CMAP_BLACK - - -def plot_leaflet_map(COMPILED_DIR, fpaths): - - """ - Convert Raster to Web-Compatible Format: - Leaflet does not directly support .TIF files, so you’ll need to convert your .TIF file to a format that Leaflet can display, - like PNG or JPEG. You can use a tool like rasterio to handle this conversion. - - Create a Web Tile Layer: - Convert the raster data into tiles (e.g., using Mapbox or a tile server). - Set Up Your Dash App: - - Create a Dash app with Dash Leaflet to display the map and raster tiles. - - """ - USA_CENTER = [39.8283, -98.5795] - - TIFPATH = os.path.join(COMPILED_DIR, fpaths[0]) - data_df, array, source_crs, geo_crs, df_coors_long, bounds, PNG = open_as_raster(TIFPATH=TIFPATH, is_reproject=False, is_convert_to_png=True) - - # TODO: - # see if can change CRS of the basemap (experiment with different basemap options) - # see if can plot the df_coors_long as an alternative with leaflet where you set the size of the point. - - # dlx.dicts_to_geojson(geo_data) - - # ## Map Update - # if len(location)>0: - # df_loc = pd.DataFrame({"state_lower":location}) - # df_loc = df_loc.groupby('state_lower').size().reset_index(name='doc_count') - # df_loc['state_lower'] = df_loc.state_lower.str.lower() - # df_loc = df_loc.merge(df_location, on='state_lower', how = 'inner') - # geo_data = df_loc.apply(lambda x: [dict(lat=x.latitude, lon=x.longitude)]*x.doc_count, axis = 1) - # geo_data = reduce(lambda x, y: x + y, geo_data) - # else: geo_data=[] - - # MAP CONTROLLER (I.E., LEGEND) - # cmaps = ["Viridis", "Spectral", "Greys"] - # lbl_map = dict(wspd="Wind speed", temp="Temperature") - # unit_map = dict(wspd="m/s", temp="°K") - # srng_map = dict(wspd=[0, 10], temp=[250, 300]) - # param0 = "temp" - # cmap0 = "Viridis" - # srng0 = srng_map[param0] - - map_fig = dl.Map( - style = - {'height': '700px', - 'width': '1500px'}, - center = USA_CENTER, - zoom = 4, - id = "leaflet-map", - children = [ - dl.LayersControl([ - dl.Overlay(dl.LayerGroup(dl.TileLayer( - # url = "https://cartodb-basemaps-{s}.global.ssl.fastly.net/light_nolabels/{z}/{x}/{y}.png", - url='https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', - id = "TileMap")), - name = "Carto DB", checked=False), - dl.Overlay(dl.LayerGroup(dl.TileLayer( - # url = 'https://api.mapbox.com/styles/v1/mapbox/dark-v10/tiles/{z}/{x}/{y}?access_token=ACCESS TOKEN' - url='https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', - )), - name = "Map Box", checked=True), - # COG fed into Tilelayer using TiTiler url (taken from r["tiles"][0]) - dl.Overlay(dl.LayerGroup(dl.TileLayer( - url = PNG, - opacity = 0.8, - id = "carbon_2000")), - name = "Carbon_Stock_2000", checked = True), - dl.LayerGroup(id = "layer"), - # Set colorbar and location in app - # dl.Colorbar(colorscale = colorscale, width = 20, height = 150, min = minv, max = maxv, position = "bottomright"), - # info, - ]) - ] - ) - - # map_fig = dl.Map(id="map", - # center=USA_CENTER, - # zoom=4, - # style={"width": "100%", "height": "100%"}, - # # crs=str(geo_crs).replace(":", ""), - # children=[ - # # dl.TileLayer(), - # dl.TileLayer( - # id="tile-layer", - # url='https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', - # # https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer/tile/{z}/{y}/{x} - # attribution='Map data © OpenStreetMap contributors', - # # opacity=0.5 - # ), - # dl.ImageOverlay(url=PNG, opacity=0.5, bounds=bounds), #tileSize=912 #width_height - # # WMSTileLayer - # # dl.WMSTileLayer(url="https://mesonet.agron.iastate.edu/cgi-bin/wms/nexrad/n0r.cgi", - # # layers="nexrad-n0r-900913", format="image/png", transparent=True), - # # dl.Colorbar(id="cbar", width=150, height=20, style={"margin-left": "40px"}, position="bottomleft"), - # ]) - - fig_div = html.Div(children=[ - # Create the map itself. - map_fig - , - # Create controller. - # html.Div(children=[ - # html.Div("Parameter"), - # dcc.Dropdown(id="dd_param", options=[dict(value=p, label=lbl_map[p]) for p in PARAMS], value=param0), - # html.Br(), - # html.Div("Colorscale"), - # dcc.Dropdown(id="dd_cmap", options=[dict(value=c, label=c) for c in cmaps], value=cmap0), - # html.Br(), - # html.Div("Opacity"), - # dcc.Slider(id="opacity", min=0, max=1, value=0.5, step=0.1, marks={0: "0", 0.5: "0.5", 1: "1"}), - # html.Br(), - # html.Div("Stretch range"), - # dcc.RangeSlider(id="srng", min=srng0[0], max=srng0[1], value=srng0, - # marks={v: "{:.1f}".format(v) for v in srng0}), - # html.Br(), - # html.Div("Value @ click position"), - # html.P(children="-", id="label"), - # ], - # className="info") - - ], - - style={"display": "grid", "width": "100%", "height": "100vh"} - ) - - # fig_div = html.Div(id="plotly-map", - # children=[dl.Map( - # # dl.BaseLayer(id="baselayer"), - # # dl.TileLayer(url=TIFPATH), center=[56,10], zoom=6, style={'height': '80vh', 'width': '80vh'} - # dl.LayersControl( - # [ - # dl.Overlay( - # [ - # # dl.ImageOverlay(url=png_path, bounds=bounds_png) - # dl.GeoTIFFOverlay( - # url=TIFPATH) #, bounds=bounds_tiff) - # ], - # name="Map", checked=True) - # ], - # position="topleft" - # ), - # ), - # # dl.GeoJSON(data=dlx.dicts_to_geojson( - # # get_map_data() - # # ), - # # cluster=True, - # # superClusterOptions={"radius": 100}, id='map-geojson'), - - - # ], - # ) - - # fig_div = html.Div(id="plotly-map", - # children=dl.Map( - # children=[ - # dl.TileLayer(), - # dl.LayersControl([dl.BaseLayer( - # dl.TileLayer(url=TIFPATH)) #, name=key, checked=key == "temp_new", id=key) for key in keys - # ] - # + - # [dl.Overlay(dl.LayerGroup(markers), name="TEST", checked=True)] - # ) - # ], - # center=[52.195573, 20.883528], zoom=12, id='map'), - # style={'width': '80vh', 'height': '80vh'}), - # # 'margin': "auto", "display": "block", "position": "relative" - - - print("Leaflet and TiTiler") - - return fig_div diff --git a/dash_app/src/mapbox_raster.py b/dash_app/src/mapbox_raster.py deleted file mode 100644 index 5aab50f..0000000 --- a/dash_app/src/mapbox_raster.py +++ /dev/null @@ -1,280 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# standard -import os - -# visualization and data manipulation -from dash import html, dcc -import plotly.express as px -import geopandas as gpd -import shapely.geometry - -# sourced scripts -from src.reader import open_as_raster -from definitions import plotly_config, CMAP_BLACK - - -def plot_mapbox_map(COMPILED_DIR, fpaths): - - # NOT GOING TO WORK - - # map box icons: https://labs.mapbox.com/maki-icons/ - - TIFPATH = os.path.join(COMPILED_DIR, fpaths[0]) - data_df, array, source_crs, geo_crs, df_coors_long, boundingbox, img = open_as_raster(TIFPATH=TIFPATH, is_reproject=True, is_convert_to_png=False) - - fig = px.scatter_mapbox(df_coors_long, lat="LatitudeProj", lon="LongitudeProj", hover_name="IsFeasible", #hover_data=["State", "Population"], - size="IsFeasible", - size_max=1, # This sets the maximum size of points - # size_max_scale=10, # Controls how much the size scales relative to size_max - color_discrete_sequence=["fuchsia"], - zoom=3, - # height=300 - ) - - - - # fig = go.Figure(go.Scattermapbox( - # lat=df_coors_long['LatitudeProj'], - # lon=df_coors_long['LongitudeProj'], - # # text=data['label'], - # mode='markers', - # marker=dict( - # color='red', - # symbol="bus", - # opacity=0.7, - # size=10 # Initial placeholder value - # ) - # )) - # fig.update_layout( - # mapbox = dict( - # accesstoken=token, - # # style="outdoors", zoom=0.7), - # # showlegend = False, - # ) - # ) - - fig.update_layout(mapbox_style="open-street-map") # Allowed values which do not require a token are 'open-street-map', 'white-bg', 'carto- positron', 'carto-darkmatter'. - fig.update_layout(margin={"r":0,"t":0,"l":0,"b":0}) - fig.update_traces(marker=dict(size=4)) - - fig.update_mapboxes(layers=[ - { "symbol": {"icon": "square"}, - "sourcetype": "geojson", - "type": "symbol", # Type: enumerated , one of ( "circle" | "line" | "fill" | "symbol" | "raster" ) - # "circle": {"radius": 13} - } - ]) - custom_javascript = html.Script(''' -document.addEventListener('DOMContentLoaded', function() { - var mapElement = document.querySelector('#map'); - if (mapElement) { - var map = mapElement._plotly_js._fullData[0].mapboxMap; - - function updatePointSize() { - var zoom = map.getZoom(); - // Calculate circle radius to cover approximately 1 km - // Size in pixels = 1 km / (meters per pixel at current zoom level) - var metersPerPixel = 156543.03392 * Math.cos(map.getCenter().lat * Math.PI / 180) / Math.pow(2, zoom); - var radiusInPixels = 1000 / metersPerPixel; // 1000 meters (1 km) - - map.setPaintProperty('my-layer', 'circle-radius', radiusInPixels); - } - - map.on('zoom', updatePointSize); - updatePointSize(); // Initial call to set size - } -}); -''') - - - # fig.update_layout( - # mapbox_style="mapbox://styles/mapbox/streets-v11", - # mapbox_layers=[ - # { - # 'sourcetype': 'geojson', - # 'source': { - # 'type': 'FeatureCollection', - # 'features': [{'type': 'Feature', 'geometry': {'type': 'Point', 'coordinates': [lon, lat]}} for lon, lat in zip(df_coors_long['LongitudeProj'], df_coors_long['LatitudeProj'])] - # }, - # 'type': 'circle', - # 'point': { - # 'circle-radius': ['case', ['==', ['zoom'], 0], 10, 10], # Static radius - # 'circle-color': '#FF0000', - # 'circle-opacity': 0.7 - # } - # } - # ] - # ) - - # fig = px.density_mapbox( - # df_coors_long, - # lat="LatitudeProj", - # lon="LongitudeProj", - # z="IsFeasible", - # radius=1, - # zoom=3, - # # color_continuous_scale=["#121212", "#121212"], - # # opacity=0.9 - # # mapbox_style="open-street-map" - # ) - # fig = go.Figure(go.Scattermapbox()) - # fig.update_layout( - # mapbox_layers=[ - # { - # # "below": "traces", - # "circle": {"radius": 3}, - # "color":"red", - # "minzoom": 6, - # "source": gpd.GeoSeries( - # df_coors_long.loc[:, ["LongitudeProj", "LatitudeProj"]].apply( - # shapely.geometry.Point, axis=1 - # ) - # ).__geo_interface__, - # }, - # ], - # mapbox_style="carto-positron", - # ) - fig.update_layout( - mapbox_layers=[ - { - # "below": "traces", - "circle": {"radius": 10}, - "color":"red", - "minzoom": 10, - "source": gpd.GeoSeries( - df_coors_long.loc[:, ["LongitudeProj", "LatitudeProj"]].apply( - shapely.geometry.Point, axis=1 - ) - ).__geo_interface__, - }, - ], - mapbox_style="open-street-map", - ) - - # Create a scatter mapbox plot - # fig = go.Figure(go.Scattermapbox( - # lat=df_coors_long['LatitudeProj'], - # lon=df_coors_long['LongitudeProj'], - # mode='markers', - # marker=dict( - - # size=3, # Fixed size for all points - # color=df_coors_long['IsFeasible'], # Optional: Color by a value - # colorscale='Viridis', # Optional: Color scale - # # symbol="circle", - # showscale=True # Optional: Show color scale - # ), - # # text=df_coors_long['IsFeasible'], # Optional: Tooltip text - # )) - - # # Update layout with Mapbox style - # fig.update_layout( - # mapbox=dict( - # style='carto-positron', # Choose a Mapbox style - # # center=dict(lat=41, lon=-75), # Center map on your data - # # zoom=10 # Initial zoom level - # ), - # # title='Map with Fixed Point Size' - # ) - - fig_div = html.Div(id="plotly-map", - children=[dcc.Graph(id="energy-map", figure=fig, config=plotly_config), - custom_javascript] - ) - - # fig.update_layout(mapbox_style="open-street-map") - - # df = px.data.gapminder().query("year == 2007") - # fig = px.scatter_geo(df, locations="iso_alpha", - # color="continent", # which column to use to set the color of markers - # hover_name="country", # column added to hover information - # size="pop", # size of markers - # projection="natural earth") - - ## IF USING MAPBOX THIS IS GREAT BUT IT DOESN'W WORK FOR IMSHOW - # Add a basemap layer (a custom image or plotly mapbox style) - optional - # fig.update_layout( - # mapbox_style="open-street-map", # Choose a basemap style (e.g., 'open-street-map', 'carto-positron', 'white-bg') - # mapbox=dict( - # center=dict(lat=37.0902, lon=-95.7129), # Center of the US (latitude and longitude) - # zoom=3 # Adjust the zoom level as needed - # ), - # autosize=True - # ) - # Add a trace in order for the base map to appear - # fig.add_trace(go.Scattermapbox( - # mode='markers', - # lon=[-95.7129], # Longitude of center (for example) - # lat=[37.0902], # Latitude of center (for example) - # marker=dict( - # size=10, # Fixed size? - # color='red' - # ) - # )) - - # Update layout with geo projection and center -- this did nothing ... - # fig.update_layout( - # geo=dict( - # scope='usa', - # center=dict(lat=39.0902, lon=-95.7129), # Center of the US - # projection_scale=5 # Adjust this as needed - # ) - # ) - # fig.add_trace(px.imshow(array)) - - - - - - - # print(data_array) - - # fig_div = html.Div(id="plotly-map", - # children=[ - # px.scatter_mapbox(data_array), # lon=data_array.x, lat=data_array.y - # ] - # ) - # fig_div.update_layout( - # title='Raster Overlay on Map', - # geo=dict( - # scope='usa', - # projection=dict(type='mercator'), - # showland=True, - # landcolor='rgb(243, 243, 243)' - # ) - # ) - - - # Convert to Plotly-compatible format - # img_trace = px.imshow(data_array) - - # # Create map with base layer and raster overlay - # fig = px.scatter_mapbox( - # lat=[37.7749], - # lon=[-95.7129], - # color_discrete_sequence=['blue'], - # zoom=4, - # mapbox_style='open-street-map' - # ) - - # print(img_trace.data[0]) - # fig.add_trace(img_trace.data[0]) - - # fig.update_layout( - # title='Raster Overlay on Map', - # mapbox=dict( - # style='open-street-map', - # center=dict(lat=37.7749, lon=-95.7129), - # zoom=4 - # ) - # ) - - # fig_div = html.Div(id="plotly-map", - # children=[dcc.Graph(id="energy-map", figure=fig, config=plotly_config)] - # ) - - print("Mapbox") - - return fig_div diff --git a/dash_app/src/reader.py b/dash_app/src/reader.py index cf30056..6dee814 100644 --- a/dash_app/src/reader.py +++ b/dash_app/src/reader.py @@ -173,115 +173,3 @@ def open_as_raster(TIFPATH, is_reproject=False, is_convert_to_png=False): # print(geo_lats.shape) return data_df, array, source_crs, geo_crs, df_coors_long, bbox, img - - - - - - - - - - - # with rasterio.open(TIF_stream) as src: - - # array = src.read(1) - # boundingbox = src.bounds - - # if is_convert_to_png: - # # TIF to IMG (i.e., PNG) - # array_blackwhite = np.interp(array, (array.min(), array.max()), (0, 255)) # 255 is white and 0 is black - # img = Image.fromarray(array_blackwhite.astype(np.uint8)).resize((src.width, src.height)) - # else: - # img = [] - - # print('here') - # print(array) - - # metadata = src.meta - - - - -def read_data(): - - # get the parent directory path to where this notebook is currently stored - root_dir = os.path.dirname(os.getcwd()) - - # data directory in repository - # data_dir = os.path.join(root_dir, "data") - data_dir = os.path.join(root_dir, "../data") - - # GRIDCERF data directory from downloaded archive - gridcerf_dir = os.path.join(data_dir, "gridcerf") - - # GRIDCERF reference data directory - reference_dir = os.path.join(gridcerf_dir, "reference") - - - - # validation directory - validation_dir = os.path.join(gridcerf_dir, 'validation') - - # common exclusion layers - raster_dir = os.path.join(gridcerf_dir, 'common') - - # directory containing technology specific layers - tech_dir = os.path.join(gridcerf_dir, 'technology_specific') - - # template land mask raster - template_raster = os.path.join(reference_dir, "gridcerf_landmask.tif") - - # additional layers needed - slope_raster = os.path.join(tech_dir, 'gridcerf_srtm_slope_20pct_or_less.tif') - potential_raster = os.path.join(tech_dir, 'gridcerf_nrel_wind_development_potential_hubheight080m_cf35.tif') - population_raster = os.path.join(tech_dir, 'gridcerf_densely_populated_ssp2_2020.tif') - - # EIA power plant spatialdata - power_plant_file = os.path.join(validation_dir, 'eia', 'accessed_2021-06-09', 'PowerPlants_US_EIA', 'PowerPlants_US_202004.shp') - - # eia generator inventory file downloaded from: https://www.eia.gov/electricity/data/eia860m/xls/april_generator2021.xlsx - eia_generator_inventory_file = os.path.join(validation_dir, 'eia', 'april_generator2021.xlsx') - - # eia 923 generation data downloaded from: https://www.eia.gov/electricity/data/eia923/archive/xls/f923_2021.zip - eia_generation_file = os.path.join(validation_dir, "eia", "EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx") - - # administrative data - cerf_conus_file = os.path.join(reference_dir, 'gridcerf_conus_boundary.shp') - cerf_states_file = os.path.join(reference_dir, 'Promod_20121028_fips.shp') - - # output combined suitability layer without land mask - cerf_suitability_file = os.path.join(validation_dir, 'temp', 'gridcerf_wind_suitability_nomask.tif') - - # combined suitability with land mask - cerf_suitability_masked_file = os.path.join(validation_dir, 'temp', 'gridcerf_wind_suitability_landmask.tif') - - # validation outputs - suitable_power_plants_file = os.path.join(validation_dir, 'results', 'suitable_power_plants_wind.shp') - unsuitable_power_plants_file = os.path.join(validation_dir, 'results', 'unsuitable_power_plants_wind.shp') - - paths_dict = {"validation_dir": validation_dir, - "raster_dir": raster_dir, - "tech_dir": tech_dir, - "template_raster": template_raster, - "slope_raster": slope_raster, - "potential_raster": potential_raster, - "population_raster": population_raster, - "power_plant_file": power_plant_file, - "eia_generator_inventory_file": eia_generator_inventory_file, - "eia_generation_file": eia_generation_file, - "cerf_conus_file": cerf_conus_file, - "cerf_states_file": cerf_states_file, - "cerf_suitability_file": cerf_suitability_file, - "cerf_suitability_masked_file": cerf_suitability_masked_file, - "suitable_power_plants_file": suitable_power_plants_file, - "unsuitable_power_plants_file": unsuitable_power_plants_file} - - - return paths_dict - - - - - - diff --git a/dash_app/src/reader_deckgl.py b/dash_app/src/reader_deckgl.py deleted file mode 100644 index cf30056..0000000 --- a/dash_app/src/reader_deckgl.py +++ /dev/null @@ -1,287 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# standard libraries -import os - -# data manipulation -import numpy as np -import pandas as pd -import rasterio -import rioxarray -import xarray as xr -from pyproj import Transformer, Proj, transform -from pyproj import CRS -from PIL import Image -# import rasterio -# from rasterio.warp import calculate_default_transform, reproject, Resampling - -# sourced scripts -from definitions import CONNECT_TO_LAMBDA -if CONNECT_TO_LAMBDA: - from definitions import DATASET_ID - from msdlive_utils import get_bytes - from io import BytesIO - -WEB_CRS = "EPSG:4326" # WGS 84 -# WEB_CRS = "EPSG:3857" # WEB MERCATOR - -def read_TIF_metadata(TIF_stream): - - # rioxarray - og_proj_array = rioxarray.open_rasterio(TIF_stream, masked=True) # returns xarray.DataArray - - # xds_lonlat = og_proj_array.rio.reproject(WEB_CRS) - # print("HERE") - # print(xds_lonlat) - # print('\n\n\n') - - data_2d = og_proj_array.isel(band=0).values - data_df = pd.DataFrame(data_2d, columns=og_proj_array.x, index=og_proj_array.y) - # og_lons = og_proj_array.x - # og_lats = og_proj_array.y - - # Metadata - # metadata = og_proj_array.rio.meta() - - # Transform - transform = og_proj_array.rio.transform() - - # Albers Equal Area Conic CRS (ESRI:102003) - source_crs = og_proj_array.rio.crs - pyproj_crs = CRS(source_crs.to_proj4()) - units_crs = pyproj_crs.axis_info[0].unit_name - - # Bounding Box - source_bounds = og_proj_array.rio.bounds() - min_lon = source_bounds[0] # minx - min_lat = source_bounds[1] # miny - max_lon = source_bounds[2] # maxx - max_lat = source_bounds[3] # maxy - - source_bbox = [[min_lat, min_lon],[max_lat, max_lon]] - - return og_proj_array, data_df, transform, source_crs, units_crs, source_bbox - - -def melt_and_reproject(array, TIF_df, source_crs, geo_crs): - - df_reset = TIF_df.reset_index() - df_reset = df_reset.rename(columns={'index': 'Latitude'}) - df_melted = pd.melt(df_reset, id_vars='Latitude', var_name='Longitude', value_name='IsFeasible') - - # Transform - transformer = Transformer.from_crs(source_crs, geo_crs, always_xy=True) - df_melted["LongitudeProj"], df_melted["LatitudeProj"] = transformer.transform(df_melted.Longitude, df_melted.Latitude) - - # Bounding Box - min_lat = df_melted['LatitudeProj'].min() - max_lat = df_melted['LatitudeProj'].max() - min_lon = df_melted['LongitudeProj'].min() - max_lon = df_melted['LongitudeProj'].max() - - bbox_proj = [[min_lat, min_lon],[max_lat, max_lon]] - - # Remove NonFeasible - df_melted_feasible = df_melted[df_melted["IsFeasible"] != 1] # (216165, 5) - - # the BELOW reprojects EVERYTHING EVEN THE BINARY (BOOLEAN, is feasible?) CELLS ... - proj_array = array.rio.reproject(geo_crs) - proj_lons = proj_array.x #np.array(proj_array.x) - proj_lats = proj_array.y # np.array(proj_array.y) - data_2d = proj_array.isel(band=0).values - data_df_proj = pd.DataFrame(data_2d, columns=proj_lons, index=proj_lats) - - unique_values = pd.unique(data_df_proj.values.flatten()) - # print(unique_values) - - return proj_array, df_melted_feasible, bbox_proj, data_df_proj - -def open_as_raster(TIFPATH, is_reproject=False, is_convert_to_png=False): - - """This should work with any file that rasterio can open - (most often: geoTIFF). The x and y coordinates are generated - automatically from the file's geoinformation, shifted to the - center of each pixel (see “PixelIsArea” Raster Space for more - information).""" - - TIF_source = "Local" - TIF_stream = TIFPATH - - if CONNECT_TO_LAMBDA: - - TIF_source = "S3" - TIF_content = get_bytes(DATASET_ID, TIFPATH) # reading the retrieved file from the S3 bucket - TIF_stream = BytesIO(TIF_content) # wrap the file content in a BytesIO object for use like a file - - print(f"{TIF_source}: {TIF_stream}\n") - - array, data_df, transform, source_crs, units_crs, source_bbox = read_TIF_metadata(TIF_stream=TIF_stream) - source_bbox = [[24.9493, -125.00165], [49.59037, -66.93457]] # bounds of the United States, not considering off shore - # source_bbox = [[19.94822477183972, -134.3417298122159], [52.7538229058337, -60.14850035217076]] - - print(f"Data is in {source_crs} and its shape is {array.shape}") - - # WGS84 (aka. Albers Equal) || destination_proj - geo_crs = CRS(WEB_CRS) - """ - dash-leaflet itself primarily supports EPSG:3857 (Web Mercator) and EPSG:4326 (WGS 84) out of the box - """ - if is_reproject: - array, df_coors_long, proj_bbox, data_df = melt_and_reproject(array=array, # og projected array - TIF_df=data_df, - source_crs=source_crs, - geo_crs=geo_crs - ) - print(f"Data is now in {geo_crs} and its shape is {array.shape}") - bbox = proj_bbox - else: - df_coors_long = [] - bbox = source_bbox - - array = array.values - array = array.reshape(-1, array.shape[-1]) - - if is_convert_to_png: - # TIF to IMG (i.e., PNG) - array_blackwhite = np.interp(array, (array.min(), array.max()), (0, 255)) # 255 is white and 0 is black - img = Image.fromarray(array_blackwhite.astype(np.uint8)).resize((array_blackwhite.shape[0], array_blackwhite.shape[1])) - else: - img = [] - - array[array == 1] = np.nan - - print(f"Bounding Box: {bbox}") - - with rasterio.open(TIF_stream) as src: - transform = src.transform - transformer = Transformer.from_crs(source_crs, geo_crs, always_xy=True) - width = src.width - height = src.height - - cols, rows = np.meshgrid(np.arange(width), np.arange(height)) - cols = cols.flatten() - rows = rows.flatten() - xs, ys = transform * (cols, rows) - geo_lons, geo_lats = transformer.transform(xs, ys) - geo_lons = geo_lons.reshape((height, width)) - geo_lats = geo_lats.reshape((height, width)) - # print(geo_lons) - # print(geo_lats) - - # 1D, 2D, 3D values - # print(geo_lats.shape) - - return data_df, array, source_crs, geo_crs, df_coors_long, bbox, img - - - - - - - - - - - # with rasterio.open(TIF_stream) as src: - - # array = src.read(1) - # boundingbox = src.bounds - - # if is_convert_to_png: - # # TIF to IMG (i.e., PNG) - # array_blackwhite = np.interp(array, (array.min(), array.max()), (0, 255)) # 255 is white and 0 is black - # img = Image.fromarray(array_blackwhite.astype(np.uint8)).resize((src.width, src.height)) - # else: - # img = [] - - # print('here') - # print(array) - - # metadata = src.meta - - - - -def read_data(): - - # get the parent directory path to where this notebook is currently stored - root_dir = os.path.dirname(os.getcwd()) - - # data directory in repository - # data_dir = os.path.join(root_dir, "data") - data_dir = os.path.join(root_dir, "../data") - - # GRIDCERF data directory from downloaded archive - gridcerf_dir = os.path.join(data_dir, "gridcerf") - - # GRIDCERF reference data directory - reference_dir = os.path.join(gridcerf_dir, "reference") - - - - # validation directory - validation_dir = os.path.join(gridcerf_dir, 'validation') - - # common exclusion layers - raster_dir = os.path.join(gridcerf_dir, 'common') - - # directory containing technology specific layers - tech_dir = os.path.join(gridcerf_dir, 'technology_specific') - - # template land mask raster - template_raster = os.path.join(reference_dir, "gridcerf_landmask.tif") - - # additional layers needed - slope_raster = os.path.join(tech_dir, 'gridcerf_srtm_slope_20pct_or_less.tif') - potential_raster = os.path.join(tech_dir, 'gridcerf_nrel_wind_development_potential_hubheight080m_cf35.tif') - population_raster = os.path.join(tech_dir, 'gridcerf_densely_populated_ssp2_2020.tif') - - # EIA power plant spatialdata - power_plant_file = os.path.join(validation_dir, 'eia', 'accessed_2021-06-09', 'PowerPlants_US_EIA', 'PowerPlants_US_202004.shp') - - # eia generator inventory file downloaded from: https://www.eia.gov/electricity/data/eia860m/xls/april_generator2021.xlsx - eia_generator_inventory_file = os.path.join(validation_dir, 'eia', 'april_generator2021.xlsx') - - # eia 923 generation data downloaded from: https://www.eia.gov/electricity/data/eia923/archive/xls/f923_2021.zip - eia_generation_file = os.path.join(validation_dir, "eia", "EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx") - - # administrative data - cerf_conus_file = os.path.join(reference_dir, 'gridcerf_conus_boundary.shp') - cerf_states_file = os.path.join(reference_dir, 'Promod_20121028_fips.shp') - - # output combined suitability layer without land mask - cerf_suitability_file = os.path.join(validation_dir, 'temp', 'gridcerf_wind_suitability_nomask.tif') - - # combined suitability with land mask - cerf_suitability_masked_file = os.path.join(validation_dir, 'temp', 'gridcerf_wind_suitability_landmask.tif') - - # validation outputs - suitable_power_plants_file = os.path.join(validation_dir, 'results', 'suitable_power_plants_wind.shp') - unsuitable_power_plants_file = os.path.join(validation_dir, 'results', 'unsuitable_power_plants_wind.shp') - - paths_dict = {"validation_dir": validation_dir, - "raster_dir": raster_dir, - "tech_dir": tech_dir, - "template_raster": template_raster, - "slope_raster": slope_raster, - "potential_raster": potential_raster, - "population_raster": population_raster, - "power_plant_file": power_plant_file, - "eia_generator_inventory_file": eia_generator_inventory_file, - "eia_generation_file": eia_generation_file, - "cerf_conus_file": cerf_conus_file, - "cerf_states_file": cerf_states_file, - "cerf_suitability_file": cerf_suitability_file, - "cerf_suitability_masked_file": cerf_suitability_masked_file, - "suitable_power_plants_file": suitable_power_plants_file, - "unsuitable_power_plants_file": unsuitable_power_plants_file} - - - return paths_dict - - - - - - diff --git a/dash_app/src/utilities.py b/dash_app/src/utilities.py index 642c0e9..ac83b4f 100644 --- a/dash_app/src/utilities.py +++ b/dash_app/src/utilities.py @@ -2,7 +2,6 @@ # -*- coding: utf-8 -*- # data manipulation -import numpy as np import pandas as pd def print_full(df): @@ -27,199 +26,4 @@ def recur_dictify(df): d = {k: recur_dictify(g.iloc[:,1:].drop_duplicates()) for k,g in grouped} - return d - -def create_datashaded_scatterplot(df): - - dataset = hv.Dataset(df) - scatter = datashade( - hv.Scatter(dataset, kdims=["x"], vdims=["y"]) - ).opts(width=800, height=800) - - return scatter - - -def update_plot(df): - - scatter = create_datashaded_scatterplot(df) - components = to_dash(app, [scatter]) - - return components.children - - - - - - - - - - - - - - - - -# if CONNECT_TO_LAMBDA: - -# def open_as_raster(TIFPATH): - -# # reading the retrieved file from the S3 bucket -# file_content = get_bytes(DATASET_ID, TIFPATH) - -# # wrap the file content in a BytesIO object for use like a file -# file_stream = BytesIO(file_content) - -# with rasterio.open(file_stream) as src: -# array = src.read(1) -# metadata = src.meta -# array_nodata = np.where(array == src.nodata, np.nan, 0) -# array = np.where(array==1, np.nan, array) - -# data_array = pd.DataFrame(array, columns=data_array.x, index=data_array.y) - -# return data_array, array, metadata - - -# else: - - - -# def rasterio_open(): - -# with rasterio.open(TIF_stream) as src: # returns rasterio.DatasetReader - -# array = src.read(1) -# metadata = src.meta - -# # orginal projected coordinates (albers_conic_crs) to geographic coordinates (geo_crs) -# # conic_crs = src.crs # Albers Equal Area Conic CRS (ESRI:102003) -# geo_crs = CRS("EPSG:4326") # WGS84 (aka. Albers Equal) -# pyproj_crs = CRS(source_crs.to_proj4()) -# proj_units = pyproj_crs.axis_info[0].unit_name -# transform = src.transform # affine transform from map pixel coors to geo coors -# transformer = Transformer.from_crs(source_crs, geo_crs, always_xy=True) -# print("\tCRS: ", source_crs) -# print("\tProjection Units: ", proj_units) -# print("\n") - - -# albers_lon, albers_lat = src.xy(2654, 2891) # row value, column value -# geo_lon, geo_lat = transformer.transform(albers_lon, albers_lat) -# coors = [geo_lat, geo_lon] - -# cols, rows = np.meshgrid(np.arange(src.width), np.arange(src.height)) -# xs, ys = rasterio.transform.xy(transform, rows, cols) -# # print(xs) -# # print(transform) -# # print(crs) -# # print(f"LONGITUDE: {lon} \tLATITUDE: {lat}") -# # lon_transformed, lat_transformed = transformer.transform(lon, lat) -# array_nodata = np.where(array == src.nodata, np.nan, 0) -# array = np.where(array==1, np.nan, array) - - - # def get_map_data(): - - # geo_data = df_location.apply(lambda x: [dict(lat=x.latitude, lon=x.longitude)]*x.Count, axis = 1) - # geo_data = reduce(lambda x, y: x + y, geo_data) - - # return geo_data - - - # print(data_array) - - # file_content = get_bytes(DATASET_ID, TIFPATH) - - # with rasterio.open(TIFPATH) as src: - # array = src.read(1) - # metadata = src.meta - # array_nodata = np.where(array == src.nodata, np.nan, 0) - # array = np.where(array==1, np.nan, array) - - # return data_array, array, metadata # ['driver', 'dtype', 'nodata', 'width', 'height', 'count', 'crs', 'transform'] - - - # def open_as_raster(TIFPATH): - - - # data_array = xr.DataArray.to_dataframe(name='data_array', - # data=data_array, - # # coords={ - - # # } - # ) - # print(data_array) - - - - # rioxarray.Coordinates(data_array.coords) - - # It combines xarray and rasterio similar to how geopandas - # combines functionality from pandas and fiona. - - - # with rasterio.open(file_stream) as src: - # array = src.read(1) - # crs = src.crs - # metadata = src.meta - # # full_array = src.values - # # lat = src.latitude.values #.max() or .min() or .mean() - # # lon = src.longitude.values - # # print(lat) - # # print(lon) - # array_nodata = np.where(array == src.nodata, np.nan, 0) - # array = np.where(array==1, np.nan, array) - - - # # use rioxarray to get lat and long of - # data_array = rioxarray.open_rasterio(file_stream, masked=True) - # lon = data_array.x - # lat = data_array.y - - - # def get_map_data(): - - # geo_data = df_location.apply(lambda x: [dict(lat=x.latitude, lon=x.longitude)]*x.Count, axis = 1) - # geo_data = reduce(lambda x, y: x + y, geo_data) - - # return geo_data - - - - - - - - - # def tif_to_png(tif_path): - # with rasterio.open(tif_path) as src: - # array = src.read(1) # read the first (and only) band - # array = np.interp(array, (array.min(), array.max()), (0, 255)) # 255 is white and 0 is black - # img = Image.fromarray(array.astype(np.uint8)).resize((src.width, src.height)) - - # # print(src) - # print(src.bounds) - # # metadata = src.meta # dict_keys(['driver', 'dtype', 'nodata', 'width', 'height', 'count', 'crs', 'transform']) - # # print(metadata['transform']) - # # print('\n') - # # print(metadata['crs']) - # # crs = src.crs - # # print() - # # lat_max = src.latitude.values.max() - # # lat_min = src.latitude.values.min() - # # lon_max = src.longitude.values.max() - # # lon_min = src.longitude.values.min() - - # # print(lat_max, lat_min, lon_max, lon_min) - # return img, src.bounds, [src.width, src.height], [-171.791110603, 18.91619, -66.96466, 71.3577635769] - - # # TIFPATH = os.path.join(COMPILED_DIR, fpath) - # # print(TIFPATH) - # # data_array, array, metadata = open_as_raster(TIFPATH=TIFPATH) - - # PNG, bounds, width_height, bbox = tif_to_png(tif_path=TIFPATH) - - # # PNG = array - # print(array.shape) \ No newline at end of file + return d \ No newline at end of file diff --git a/dash_app/test.py b/dash_app/test.py deleted file mode 100644 index cf9fa68..0000000 --- a/dash_app/test.py +++ /dev/null @@ -1,169 +0,0 @@ -# import dash_leaflet as dl -# from dash import Dash - -# # Cool, dark tiles by Stadia Maps. -# url = 'https://tiles.stadiamaps.com/tiles/alidade_smooth_dark/{z}/{x}/{y}{r}.png' -# attribution = '© Stadia Maps ' -# # Create app. -# app = Dash() -# app.layout = dl.Map([ -# dl.TileLayer(url=url, maxZoom=20, attribution=attribution) -# ], center=[56, 10], zoom=6, style={'height': '50vh'}) - -# if __name__ == '__main__': -# app.run_server(debug=True) - - -#Build Dash app (note: JupyterDash is being used here but will work in external dash app , to see this change last line to "mode=External") - -from jupyter_dash import JupyterDash -from dash import Dash, html,dcc,Input, Output -import dash_leaflet as dl -import json -import httpx - -# read data -S3_URL = open("Resource_Files\S3_URL.txt").read() # this is my S3 bucket URL saved in txt file for privacy - - -# Extract min and max values of the COG , this is used to rescale the COG - -titiler_endpoint = "http://localhost:8080" # titiler docker image running on local . -url = S3_URL - - -r = httpx.get( - f"{titiler_endpoint}/cog/statistics", - params = { - "url": url, - } -).json() - -minv = (r["1"]["min"]) -maxv = (r["1"]["max"]) - - -#get tile map from titiler endpoint and url , and scale to min and max values - -r = httpx.get( - f"{titiler_endpoint}/cog/tilejson.json", - params = { - "url": url, - "rescale": f"{minv},{maxv}", - "colormap_name": "viridis" - } -).json() - - - -#create function which uses "get point" via titiler to get point value. This will be used in dash to get -#point value and present this on the map - - -def get_point_value(lat,lon): - - Point = httpx.get( - f"{titiler_endpoint}/cog/point/{lon},{lat}", - params = { - "url": url, - "resampling": "average" - - } - ).json() - - - return "{:.2f}".format(float(Point["values"][0])) - - - -app = JupyterDash(__name__) - -#create the info panel where our wind speed will be displayed -info = html.Div( id="info", className="info", - style={"position": "absolute", "bottom": "10px", "left": "10px", "z-index": "1000"}) - -#Create app layout -#dash leaflet used to create map (https://dash-leaflet.herokuapp.com/) -#dl.LayersControl, dl.overlay & dl.LayerGroup used to add layer selection functionality -app.layout = html.Div([ -dl.Map(style={'width': '1000px', 'height': '500px'}, - center=[55, -4], - zoom=5, - id = "map", - children=[ - dl.LayersControl([ - - dl.Overlay(dl.LayerGroup(dl.TileLayer(url="https://cartodb-basemaps-{s}.global.ssl.fastly.net/light_nolabels/{z}/{x}/{y}.png",id="TileMap")),name="BaseMap",checked=True), - #COG fed into Tilelayer using TiTiler url (taken from r["tiles"][0]) - dl.Overlay(dl.LayerGroup(dl.TileLayer(url=r["tiles"][0], opacity=0.8,id="WindSpeed@100m")),name="WS@100m",checked=True), - dl.LayerGroup(id="layer"), - # set colorbar and location in app - dl.Colorbar(colorscale="viridis", width=20, height=150, min=minv, max=maxv,unit='m/s',position="bottomright"), - info, - -]) -]) -]) -# create callback which uses inbuilt "click_lat_lng" feature of Dash leaflet map, extract lat/lon and feed to get_point_value function , WS fed to output id = info (which is info html.div) -@app.callback(Output("info", "children"), [Input("map", "click_lat_lng")]) -def map_click(click_lat_lng): - - lat= click_lat_lng[0] - lon=click_lat_lng[1] - - return get_point_value(lat,lon) - - - -app.run_server(debug=True,mode='inline') - - - - - - - -# # Map layers -# keys = ["temp_new", "clouds_new", "precipitation_new", "pressure_new", "wind_new"] - -# #Color scales -# map_colorscale = {'temp': ['rgba(130, 87, 219, 1)', 'rgba(32, 140, 236, 1)', 'rgba(32, 196, 232, 1)', 'rgba(35, 221, 221, 1)', -# 'rgba(194, 255, 40, 1)', 'rgba(255, 240, 40, 1)', 'rgba(255, 194, 40,1)', 'rgba(252, 128, 20, 1)'], -# 'clouds': ['rgba(255, 255, 255, 0.0)', 'rgba(253, 253, 255, 0.1)', 'rgba(252, 251, 255, 0.2)', -# 'rgba(250, 250, 255, 0.3)', 'rgba(249, 248, 255, 0.4)', 'rgba(247, 247, 255, 0.5)', -# 'rgba(246, 245, 255, 0.75)', 'rgba(244, 244, 255, 1)', 'rgba(243, 242, 255, 1)', -# 'rgba(242, 241, 255, 1)', 'rgba(240, 240, 255, 1)'] - -# markers = [dl.Marker(position=[52.195573, 20.883528])] - -# #Colorbars -# colorbar_temp = dl.Colorbar(colorscale=map_colorscale['temp'], id='temp_colorbar', -# position='bottomleft', width=200, height=10, min=-30, max=30, -# nTicks=8, opacity=0.9, tickValues=[-30, -20, -10, 0, 10, 20, 30]) -# colorbar_clouds = dl.Colorbar(colorscale=map_colorscale['clouds'], id='clouds_colorbar', -# position='bottomleft', width=200, height=10, min=0, max=100, -# nTicks=11, opacity=0.9, tickValues=[0, 20, 40, 60, 80, 100]) - -# # ----------------- - -# html.Div( -# dl.Map(children=[dl.TileLayer(), -# dl.LayersControl( -# [dl.BaseLayer( -# dl.TileLayer(url=url.format(key)), name=key, checked=key == "temp_new", id=key) for key in keys] -# + -# [dl.Overlay(dl.LayerGroup(markers), name="TEST", checked=True)] -# ) -# ], center=[52.195573, 20.883528], zoom=12, id='map' ) -# , style={'width': '90%', 'height': '50vh', 'margin': "auto", "display": "block", "position": "relative"}), - -# # ------------- -# @app.callback( -# Output('???', '???'), -# Input('???', '???') -# ) -# def update_colorbar(key): -# if key == 'temp_new': -# return dl.Overlay(colorbar_temp) -# else: -# return dl.Overlay(colorbar_clouds) diff --git a/dash_app/test2.py b/dash_app/test2.py deleted file mode 100644 index 11753e6..0000000 --- a/dash_app/test2.py +++ /dev/null @@ -1,17 +0,0 @@ -import plotly.graph_objects as go - -token = open("../../mapbox_token.py").read() - -fig = go.Figure(go.Scattermapbox( - mode = "markers+text+lines", - lon = [-75, -80, -50], lat = [45, 20, -20], - marker = {'size': 20, 'symbol': ["triangle", "harbor", "airport"]}, - text = ["Bus", "Harbor", "airport"],textposition = "bottom right")) - -fig.update_layout( - mapbox = { - 'accesstoken': token, - 'style': "outdoors", 'zoom': 0.7}, - showlegend = False) - -fig.show() \ No newline at end of file diff --git a/dash_app/test3.py b/dash_app/test3.py deleted file mode 100644 index 1d04f6c..0000000 --- a/dash_app/test3.py +++ /dev/null @@ -1,95 +0,0 @@ -import dash_leaflet as dl -from dash import Dash - -# Custom icon as per official docs https://leafletjs.com/examples/custom-icons/ -custom_icon = dict( - iconUrl='https://leafletjs.com/examples/custom-icons/leaf-green.png', - shadowUrl='https://leafletjs.com/examples/custom-icons/leaf-shadow.png', - iconSize=[38, 95], - shadowSize=[50, 64], - iconAnchor=[22, 94], - shadowAnchor=[4, 62], - popupAnchor=[-3, -76] -) -# Small example app. -app = Dash() -app.layout = dl.Map([ - dl.TileLayer(), - dl.Marker(position=[55, 10]), - dl.Marker(position=[57, 10], icon=custom_icon), -], center=[56, 10], zoom=6, style={'height': '50vh'}) - -if __name__ == '__main__': - app.run_server() - - -# import dash -# from dash import dcc, html -# import dash_leaflet as dl - -# app = dash.Dash(__name__) - -# # Sample data -# markers = [ -# {"position": [37.7749, -122.4194], "label": "San Francisco"}, -# {"position": [34.0522, -118.2437], "label": "Los Angeles"}, -# {"position": [40.7128, -74.0060], "label": "New York"} -# ] - -# # Create markers for the map -# marker_components = [ -# dl.Marker(position=marker['position'], children=[ -# dl.Tooltip(marker['label']) -# ]) for marker in markers -# ] - -# app.layout = html.Div([ -# dcc.Store(id='map-config', data={ -# 'initial_zoom': 13, -# 'initial_center': [37.7749, -122.4194] # San Francisco coordinates -# }), -# dl.Map(id='map', center=[37.7749, -122.4194], zoom=13, children=[ -# dl.TileLayer(), -# *marker_components -# ]), -# html.Script(''' -# document.addEventListener('DOMContentLoaded', function() { -# var map = document.querySelector('#map')._leaflet_map; -# var markers = map._layers; - -# function updateMarkerSizes() { -# var zoom = map.getZoom(); -# var sizeInPixels = 10; // Desired constant size in pixels - -# for (var id in markers) { -# var marker = markers[id]; -# if (marker instanceof L.Marker) { -# // Update marker icon size based on the zoom level -# marker.setIcon(L.divIcon({ -# className: 'custom-marker', -# html: '
' -# })); -# } -# } -# } - -# // Update marker size when zoom changes -# map.on('zoomend', updateMarkerSizes); -# updateMarkerSizes(); // Initial call to set sizes -# }); -# '''), -# # html.Style(''' -# # .custom-marker { -# # width: 10px; /* Change this to your desired size */ -# # height: 10px; /* Change this to your desired size */ -# # background: red; -# # border-radius: 50%; -# # text-align: center; -# # line-height: 10px; /* Change this to your desired size */ -# # color: white; -# # } -# # ''') -# ]) - -# if __name__ == '__main__': -# app.run_server(debug=True) \ No newline at end of file diff --git a/dash_app/test4.py b/dash_app/test4.py deleted file mode 100644 index 2fb12ad..0000000 --- a/dash_app/test4.py +++ /dev/null @@ -1,81 +0,0 @@ -import httpx -import dash_leaflet as dl -from dash import Dash, html - -# Convert raster to Cloud-optimized GeoTiff -# gdal_translate input.tif output_cog.tif -of COG -co COMPRESS=LZW - - -# Set up titiler endpoint and url to COG -titiler_endpoint = "https://titiler.xyz" -url = "https://storage.googleapis.com/landcover_classes/agb_cog.tif" - -# Fetch File Metadata to get min/max rescaling values -r = httpx.get( - f"{titiler_endpoint}/cog/statistics", - params = { - "url": url, - } -).json() - -# Get minimum and maximum raster values -minv = (r["b1"]["min"]) # b1: band 1 -maxv = (r["b1"]["max"]) - -# Get tile map from titiler endpoint and url, and scale to min and max values -r = httpx.get( - f"{titiler_endpoint}/cog/tilejson.json", - params = { - "url": url, - "rescale": f"{minv},{maxv}", - "colormap_name": "magma" - } -).json() - -# print(r) -print(r["tiles"][0]) - - - -app = Dash(__name__) - -# colorscale = ['#000004', '#3b0f70', '#8c2981', '#de4968', '#fe9f6d', '#fcfdbf'] -colorscale = ["#000000"] - -# Create app layout -app.layout = html.Div([ - dl.Map( - style = - {'height': '700px', - 'width': '1500px'}, - center = [5.814379, 0.843645], - zoom = 12, - id = "leaflet-map", - children = [ - dl.LayersControl([ - dl.Overlay(dl.LayerGroup(dl.TileLayer( - # url = "https://cartodb-basemaps-{s}.global.ssl.fastly.net/light_nolabels/{z}/{x}/{y}.png", - url='https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', - id = "TileMap")), - name = "Carto DB", checked=False), - dl.Overlay(dl.LayerGroup(dl.TileLayer( - # url = 'https://api.mapbox.com/styles/v1/mapbox/dark-v10/tiles/{z}/{x}/{y}?access_token=ACCESS TOKEN' - url='https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', - )), - name = "Map Box", checked=True), - # COG fed into Tilelayer using TiTiler url (taken from r["tiles"][0]) - dl.Overlay(dl.LayerGroup(dl.TileLayer( - url = r["tiles"][0], - opacity = 0.8, - id = "carbon_2000")), - name = "Carbon_Stock_2000", checked = True), - dl.LayerGroup(id = "layer"), - # Set colorbar and location in app - # dl.Colorbar(colorscale = colorscale, width = 20, height = 150, min = minv, max = maxv, position = "bottomright"), - # info, - ]) - ] - ) -]) -if __name__ == '__main__': - app.run_server(debug=True) \ No newline at end of file diff --git a/lambda_function.py b/lambda_function.py index 6e4cc87..6ee8103 100644 --- a/lambda_function.py +++ b/lambda_function.py @@ -1,17 +1,13 @@ from typing import Dict, Any import json import sys - from functools import lru_cache - from apig_wsgi import make_lambda_handler -# from dash_app.app import app -from dash_app.layout import create_app - +# SOURCED SCRIPT +from dash_app.app import app +# from dash_app.layout import create_app -# https://gridcerf.dev.msdlive.org/ -# http://127.0.0.1:8060/ @lru_cache(maxsize=5) def get_wsgi_handler(): """Wrap this method in an lru_cache so if the lambda container is reused, the handler @@ -21,7 +17,8 @@ def get_wsgi_handler(): _type_: lambda handler that is wsgi compatible and backed by the Dash application """ return make_lambda_handler( - wsgi_app=create_app().server, + # wsgi_app=create_app().server, + wsgi_app=app.server, binary_support=True, ) diff --git a/requirements.txt b/requirements.txt index 130b6dc..eca8ae5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ bokeh==3.3.4 boto3==1.34.112 botocore==1.34.112 Brotli==1.1.0 +cachelib==0.9.0 certifi==2023.11.17 cffi==1.16.0 charset-normalizer==3.3.2 @@ -38,6 +39,7 @@ dash-resizable-panels==0.1.0 dash-svg==0.0.12 dash-table==5.0.0 dash_daq==0.5.0 +dash_deck==0.0.1 dask==2024.2.0 datashader==0.16.0 debugpy==1.8.1 @@ -49,6 +51,7 @@ fastjsonschema==2.19.1 fastparquet==2024.2.0 fiona==1.9.5 Flask==3.0.1 +Flask-Caching==2.3.0 Flask-Compress==1.15 fonttools==4.47.2 fqdn==1.5.1 @@ -122,6 +125,7 @@ ptyprocess==0.7.0 pure-eval==0.2.2 pycparser==2.21 pyct==0.5.0 +pydeck==0.9.1 Pygments==2.17.2 pyparsing==3.1.1 pyproj==3.6.1