Merge pull request #229 from podaac/release/2.8.0

Release/2.8.0

.github/workflows/build-pipeline.yml

@@ -166,7 +166,7 @@ jobs:
           git push origin "${{ env.software_version }}"
       - name: Publish UMM-S with new version
         id: publish-umm-s
-        uses: podaac/cmr-umm-updater@0.5.0
+        uses: podaac/cmr-umm-updater@0.6.0
         if: |
           github.ref == 'refs/heads/main'    ||
           startsWith(github.ref, 'refs/heads/release')
@@ -190,7 +190,7 @@ jobs:
           sleep 120
       - name: Publish UMM-S with new version retry
         id: publish-umm-s-retry
-        uses: podaac/cmr-umm-updater@0.5.0
+        uses: podaac/cmr-umm-updater@0.6.0
         if: |
           steps.publish-umm-s.outcome == 'failure'
         with:

CHANGELOG.md

@@ -13,6 +13,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Security
 
 
+## [2.8.0]
+### Added
+### Changed
+- Upgraded `cmr-umm-updater` to 0.6.0
+### Deprecated 
+### Removed
+### Fixed
+- Fix shapefile subsetting if there is more than a 2d in lon lat in shapefile that includes a third dimension
+- [pull/227](https://github.com/podaac/l2ss-py/pull/227): Fix null time values in TEMPO results when spatial+temporal subsetting
+- [pull/231](https://github.com/podaac/l2ss-py/pull/231): Improve fix for null time values by ensuring null valuesdon't persist on edges of longest row of True values
+### Security
+- Updated dependency versions to latest possible
+
 ## [2.7.0]
 ### Added
 ### Changed

podaac/subsetter/subset.py

@@ -1009,7 +1009,7 @@ def subset_with_shapefile(dataset: xr.Dataset,
         shapefile_df.geometry = shapefile_df['geometry'].apply(translate_longitude)
 
     # Mask and scale shapefile
-    def scale(lon, lat):
+    def scale(lon, lat, extra=None):  # pylint: disable=unused-argument
         lon = tuple(map(functools.partial(apply_scale_offset, lon_scale, lon_offset), lon))
         lat = tuple(map(functools.partial(apply_scale_offset, lat_scale, lat_offset), lat))
         return lon, lat
@@ -1212,11 +1212,11 @@ def subset(file_to_subset: str, bbox: np.ndarray, output_file: str,
 
     if min_time or max_time:
         args['decode_times'] = True
-        # check fill value and dtype, we know that this will cause an integer Overflow with xarray
-        if 'time' in nc_dataset.variables.keys():
+        # check fill value and dtype; we know that this will cause an integer Overflow with xarray
+        for time_variable in [v for v in nc_dataset.variables.keys() if 'time' in v]:
             try:
-                if nc_dataset['time'].getncattr('_FillValue') == nc.default_fillvals.get('f8') and \
-                 nc_dataset['time'].dtype == 'float64':
+                if nc_dataset[time_variable].getncattr('_FillValue') == nc.default_fillvals.get('f8') and \
+                 (nc_dataset[time_variable].dtype == 'float64') or (nc_dataset[time_variable].dtype == 'float32'):
                     args['mask_and_scale'] = True
             except AttributeError:
                 pass

podaac/subsetter/xarray_enhancements.py

@@ -233,7 +233,8 @@ def where(dataset: xr.Dataset, cond: Union[xr.Dataset, xr.DataArray], cut: bool)
                 dim_name: dim_value for dim_name, dim_value in indexers.items()
                 if dim_name in dataset[variable_name].dims
             }
-            var_cond = cond.sel({missing_dim: 1}).isel(**var_indexers)
+
+            var_cond = cond.any(axis=cond.dims.index(missing_dim)).isel(**var_indexers)
             indexed_var = dataset[variable_name].isel(**var_indexers)
             new_dataset[variable_name] = indexed_var.where(var_cond)
             variable = new_dataset[variable_name]

pyproject.toml

@@ -12,7 +12,7 @@
 
 [tool.poetry]
 name = "l2ss-py"
-version = "2.7.0"
+version = "2.8.0rc3"
 description = "L2 Subsetter Service"
 authors = ["podaac-tva <[email protected]>"]
 license = "Apache-2.0"
@@ -35,19 +35,19 @@ Shapely = "^2.0.2"
 harmony-service-lib = { version = "^1.0.24", optional = true }
 pystac = { version = "^0.5.3", optional = true }
 julian = "^0.14"
-importlib-metadata = "^6.8.0"
+importlib-metadata = "^7.0.1"
 h5py = "^3.6.0"
 cf-xarray = "*"
-numpy = "1.26.2"
+numpy = "^1.26.3"
 
 [tool.poetry.dev-dependencies]
 pytest = "~7"
-flake8 = "^6.1.0"
+flake8 = "^7.0.0"
 pytest-cov = "~4"
 pylint = "^3.0.2"
 sphinx = "^7.2.6"
 pytest-benchmark = "~4"
-moto = "4.2.10"
+moto = "4.2.13"
 jsonschema = "^4.20.0"
 m2r2 = "^0.3.2"
 sphinx-rtd-theme = "^2.0.0"

tests/test_subset.py

@@ -1421,6 +1421,33 @@ def test_duplicate_dims_tempo_ozone(data_dir, subset_output_dir, request):
         assert variable.shape == \
                out_nc.groups['support_data'].variables[var_name].shape
 
+def test_no_null_time_values_in_time_and_space_subset_for_tempo(data_dir, subset_output_dir, request):
+    """
+    Check if TEMPO time variable has no null values when subsetting by time and space simultaneously.
+    """
+    TEMPO_dir = join(data_dir, 'TEMPO')
+    tempo_no2_file = 'TEMPO_NO2_L2_V01_20231206T131913Z_S002G04.nc'
+
+    bbox = np.array(((-87, -83), (28.5, 33.7)))
+    output_file = "{}_{}".format(request.node.name, tempo_no2_file)
+    shutil.copyfile(
+        os.path.join(TEMPO_dir, tempo_no2_file),
+        os.path.join(subset_output_dir, tempo_no2_file)
+    )
+    _ = subset.subset(
+        file_to_subset=join(subset_output_dir, tempo_no2_file),
+        min_time="2023-12-06T13:00:00",
+        max_time="2023-12-06T15:00:00",
+        bbox=bbox,
+        output_file=join(subset_output_dir, output_file)
+    )
+    # check if the box_test is
+
+    in_nc = nc.Dataset(join(TEMPO_dir, tempo_no2_file))
+    out_nc = nc.Dataset(join(subset_output_dir, output_file))
+
+    assert np.isnan(out_nc.groups['geolocation']['time'][:]).any() == False
+
 
 def test_omi_novars_subset(data_dir, subset_output_dir, request):
     """