Merge pull request #287 from NREL/gb/multi_excl_file

Gb/multi excl file

reV/config/supply_curve_configs.py

@@ -6,13 +6,13 @@
 
 @author: gbuster
 """
-import h5py
 import os
 import logging
 
 from reV.utilities.exceptions import ConfigError, PipelineError
 from reV.config.base_analysis_config import AnalysisConfig
 from reV.pipeline.pipeline import Pipeline
+from rex.multi_file_resource import MultiFileResource
 
 logger = logging.getLogger(__name__)
 
@@ -43,8 +43,13 @@ def __init__(self, config):
 
     def _sc_agg_preflight(self):
         """Perform pre-flight checks on the SC agg config inputs"""
-        with h5py.File(self.excl_fpath, mode='r') as f:
-            dsets = list(f)
+
+        paths = self.excl_fpath
+        if isinstance(self.excl_fpath, str):
+            paths = [self.excl_fpath]
+
+        with MultiFileResource(paths, check_files=False) as res:
+            dsets = res.datasets
 
         if self.tm_dset not in dsets and self.res_fpath is None:
             raise ConfigError('Techmap dataset "{}" not found in exclusions '
@@ -54,16 +59,8 @@ def _sc_agg_preflight(self):
 
     @property
     def excl_fpath(self):
-        """Get the exclusions filepath"""
-
-        fpath = self['excl_fpath']
-
-        if fpath == 'PIPELINE':
-            fpath = Pipeline.parse_previous(
-                self.dirout, 'aggregation', target='fpath',
-                target_module='exclusions')[0]
-
-        return fpath
+        """Get the exclusions filepath(s)"""
+        return self['excl_fpath']
 
     @property
     def gen_fpath(self):

reV/handlers/exclusions.py

@@ -6,10 +6,11 @@
 import json
 import numpy as np
 
-from reV.utilities.exceptions import HandlerKeyError
+from reV.utilities.exceptions import HandlerKeyError, MultiFileExclusionError
 
 from rex.utilities.parse_keys import parse_keys
 from rex.resource import Resource
+from rex.multi_file_resource import MultiFileResource
 
 logger = logging.getLogger(__name__)
 
@@ -22,14 +23,26 @@ def __init__(self, h5_file, hsds=False):
         """
         Parameters
         ----------
-        h5_file : str
-            .h5 file containing exclusion layers and techmap
+        h5_file : str | list | tuple
+            .h5 file containing exclusion layers and techmap,
+            or a list of h5 files
         hsds : bool
             Boolean flag to use h5pyd to handle .h5 'files' hosted on AWS
             behind HSDS
         """
+
         self.h5_file = h5_file
-        self._h5 = Resource(h5_file, hsds=hsds)
+
+        if isinstance(h5_file, str):
+            self._h5 = Resource(h5_file, hsds=hsds)
+        elif isinstance(h5_file, (list, tuple)):
+            self._h5 = MultiFileResource(h5_file, check_files=False)
+            self._preflight_multi_file()
+        else:
+            msg = ('Expected str, list, or tuple for h5_file input but '
+                   'received {}'.format(type(h5_file)))
+            logger.error(msg)
+            raise TypeError(msg)
 
         self._iarr = None
 
@@ -65,6 +78,48 @@ def __getitem__(self, keys):
     def __contains__(self, layer):
         return layer in self.layers
 
+    def _preflight_multi_file(self):
+        """Run simple multi-file exclusion checks."""
+        lat_shape = self.h5.shapes['latitude']
+        lon_shape = self.h5.shapes['longitude']
+        for layer in self.layers:
+            lshape = self.h5.shapes[layer]
+            lshape = lshape[1:] if len(lshape) > 2 else lshape
+            if lshape != lon_shape or lshape != lat_shape:
+                msg = ('Shape of layer "{}" is {} which does not match '
+                       'latitude and longitude shapes of {} and {}. '
+                       'Check your exclusion file inputs: {}'
+                       .format(layer, self.h5.shapes[layer],
+                               lat_shape, lon_shape, self.h5._h5_files))
+                logger.error(msg)
+                raise MultiFileExclusionError(msg)
+
+        check_attrs = ('height', 'width', 'crs', 'transform')
+        base_profile = {}
+        for fp in self.h5_file:
+            with ExclusionLayers(fp) as f:
+                if not base_profile:
+                    base_profile = f.profile
+                else:
+                    for attr in check_attrs:
+                        if attr not in base_profile or attr not in f.profile:
+                            msg = ('Multi-file exclusion inputs from {} '
+                                   'dont have profiles with height, width, '
+                                   'crs, and transform: {} and {}'
+                                   .format(self.h5_file, base_profile,
+                                           f.profile))
+                            logger.error(msg)
+                            raise MultiFileExclusionError(msg)
+
+                        if base_profile[attr] != f.profile[attr]:
+                            msg = ('Multi-file exclusion inputs from {} '
+                                   'dont have matching "{}": {} and {}'
+                                   .format(self.h5_file, attr,
+                                           base_profile[attr],
+                                           f.profile[attr]))
+                            logger.error(msg)
+                            raise MultiFileExclusionError(msg)
+
     def close(self):
         """
         Close h5 instance
@@ -78,7 +133,7 @@ def h5(self):
 
         Returns
         -------
-        h5 : rex.Resource
+        h5 : rex.MultiFileResource | rex.Resource
         """
         return self._h5
 

reV/supply_curve/aggregation.py

@@ -33,8 +33,9 @@ def __init__(self, excl_fpath, excl_dict=None, area_filter_kernel='queen',
         """
         Parameters
         ----------
-        excl_fpath : str
-            Filepath to exclusions h5 with techmap dataset.
+        excl_fpath : str | list | tuple
+            Filepath to exclusions h5 with techmap dataset
+            (can be one or more filepaths).
         excl_dict : dict, optional
             Dictionary of exclusion LayerMask arugments {layer: {kwarg: value}}
             by default None
@@ -93,8 +94,9 @@ def __init__(self, excl_fpath, h5_fpath, excl_dict=None,
         """
         Parameters
         ----------
-        excl_fpath : str
-            Filepath to exclusions h5 with techmap dataset.
+        excl_fpath : str | list | tuple
+            Filepath to exclusions h5 with techmap dataset
+            (can be one or more filepaths).
         h5_fpath : str
             Filepath to .h5 file to be aggregated
         excl_dict : dict, optional
@@ -141,8 +143,9 @@ def __init__(self, excl_fpath, tm_dset, excl_dict=None,
         """
         Parameters
         ----------
-        excl_fpath : str
-            Filepath to exclusions h5 with techmap dataset.
+        excl_fpath : str | list | tuple
+            Filepath to exclusions h5 with techmap dataset
+            (can be one or more filepaths).
         tm_dset : str
             Dataset name in the techmap file containing the
             exclusions-to-resource mapping data.
@@ -216,8 +219,9 @@ def _get_excl_area(excl_fpath, excl_area=None):
 
         Parameters
         ----------
-        excl_fpath : str
-            Filepath to exclusions h5 with techmap dataset.
+        excl_fpath : str | list | tuple
+            Filepath to exclusions h5 with techmap dataset
+            (can be one or more filepaths).
         excl_area : float, optional
             Area of an exclusion pixel in km2. None will try to infer the area
             from the profile transform attribute in excl_fpath, by default None
@@ -250,8 +254,9 @@ def _extract_inclusion_mask(excl_fpath, tm_dset, excl_dict=None,
 
         Parameters
         ----------
-        excl_fpath : str
-            Filepath to exclusions h5 with techmap dataset.
+        excl_fpath : str | list | tuple
+            Filepath to exclusions h5 with techmap dataset
+            (can be one or more filepaths).
         tm_dset : str
             Dataset name in the techmap file containing the
             exclusions-to-resource mapping data.
@@ -274,9 +279,9 @@ def _extract_inclusion_mask(excl_fpath, tm_dset, excl_dict=None,
         logger.info('Pre-extracting full exclusion mask, this could take '
                     'up to 30min for a large exclusion config...')
         with ExclusionMaskFromDict(excl_fpath, layers_dict=excl_dict,
-                                   check_layers=True, min_area=min_area,
+                                   check_layers=False, min_area=min_area,
                                    kernel=area_filter_kernel) as f:
-            inclusion_mask = f.mask
+            inclusion_mask = f._generate_mask(..., check_layers=True)
             tm_mask = f._excl_h5[tm_dset] == -1
             inclusion_mask[tm_mask] = 0
 
@@ -428,8 +433,9 @@ def run_serial(cls, sc_point_method, excl_fpath, tm_dset,
         ----------
         sc_point_method : method
             Supply Curve Point Method to operate on a single SC point.
-        excl_fpath : str
-            Filepath to exclusions h5 with techmap dataset.
+        excl_fpath : str | list | tuple
+            Filepath to exclusions h5 with techmap dataset
+            (can be one or more filepaths).
         tm_dset : str
             Dataset name in the exclusions file containing the
             exclusions-to-resource mapping data.
@@ -658,8 +664,9 @@ def run(cls, excl_fpath, tm_dset, sc_point_method, excl_dict=None,
 
         Parameters
         ----------
-        excl_fpath : str
-            Filepath to exclusions h5 with techmap dataset.
+        excl_fpath : str | list | tuple
+            Filepath to exclusions h5 with techmap dataset
+            (can be one or more filepaths).
         tm_dset : str
             Dataset name in the techmap file containing the
             exclusions-to-resource mapping data.
@@ -728,8 +735,9 @@ def __init__(self, excl_fpath, h5_fpath, tm_dset, *agg_dset,
         """
         Parameters
         ----------
-        excl_fpath : str
-            Filepath to exclusions h5 with techmap dataset.
+        excl_fpath : str | list | tuple
+            Filepath to exclusions h5 with techmap dataset
+            (can be one or more filepaths).
         h5_fpath : str
             Filepath to .h5 file to aggregate
         tm_dset : str
@@ -813,8 +821,9 @@ def run_serial(cls, excl_fpath, h5_fpath, tm_dset, *agg_dset,
 
         Parameters
         ----------
-        excl_fpath : str
-            Filepath to exclusions h5 with techmap dataset.
+        excl_fpath : str | list | tuple
+            Filepath to exclusions h5 with techmap dataset
+            (can be one or more filepaths).
         h5_fpath : str
             Filepath to .h5 file to aggregate
         tm_dset : str
@@ -1116,8 +1125,9 @@ def run(cls, excl_fpath, h5_fpath, tm_dset, *agg_dset,
 
         Parameters
         ----------
-        excl_fpath : str
-            Filepath to exclusions h5 with techmap dataset.
+        excl_fpath : str | list | tuple
+            Filepath to exclusions h5 with techmap dataset
+            (can be one or more filepaths).
         h5_fpath : str
             Filepath to .h5 file to aggregate
         tm_dset : str

reV/supply_curve/cli_sc_aggregation.py

@@ -18,7 +18,7 @@
 
 from rex.utilities.hpc import SLURM
 from rex.utilities.cli_dtypes import (STR, INT, FLOAT, STRLIST, FLOATLIST,
-                                      STRFLOAT)
+                                      STRFLOAT, STR_OR_LIST)
 from rex.utilities.loggers import init_mult
 from rex.utilities.utilities import dict_str_load, get_class_properties
 
@@ -169,8 +169,9 @@ def from_config(ctx, config_file, verbose):
 
 
 @main.group(invoke_without_command=True)
-@click.option('--excl_fpath', '-exf', type=STR, required=True,
-              help='Exclusions file (.h5).')
+@click.option('--excl_fpath', '-exf', type=STR_OR_LIST, required=True,
+              help='Single exclusions file (.h5) or a '
+              'list of exclusion files (.h5, .h5).')
 @click.option('--gen_fpath', '-gf', type=STR, required=True,
               help='reV generation/econ output file.')
 @click.option('--tm_dset', '-tm', type=STR, required=True,
@@ -320,11 +321,22 @@ def direct(ctx, excl_fpath, gen_fpath, tm_dset, econ_fpath, res_fpath,
         init_mult(name, log_dir, modules=[__name__, 'reV', 'rex'],
                   verbose=verbose)
 
-        with h5py.File(excl_fpath, mode='r') as f:
-            dsets = list(f)
+        dsets = []
+        paths = excl_fpath
+        if isinstance(excl_fpath, str):
+            paths = [excl_fpath]
+        for fp in paths:
+            with h5py.File(fp, mode='r') as f:
+                dsets += list(f)
 
         if tm_dset in dsets:
             logger.info('Found techmap "{}".'.format(tm_dset))
+        elif tm_dset not in dsets and not isinstance(excl_fpath, str):
+            msg = ('Could not find techmap dataset "{}" and cannot run '
+                   'techmap with arbitrary multiple exclusion filepaths '
+                   'to write to: {}'.format(tm_dset, excl_fpath))
+            logger.error(msg)
+            raise RuntimeError(msg)
         else:
             logger.info('Could not find techmap "{}". Running techmap module.'
                         .format(tm_dset))