Ecse 1412 - just re-arrainging the schema, broke some CMR tests that …

…were fixed. (#1980)

* ECSE-1412 putting titles onto oneOf and reorganizing OrbitParameters to use oneOf instead of anyOf, but keeping the CMR record in/outputs the same.

* ECSE-1412 reorganizing FileDistributionInformationType and FileArchiveInformationType to see if CMR can handle it.

* ECSE-1412 Reverting FileArchiveInformationType and FileDistributionInformationType, but keeping titles for each sub-schema.

* ECSE-1412 updating UMM testing code

* ECSE-1412 Fixing bogus test.

* ECSE-1412 combining if and let

* ECSE-1412 replacing single letter variables.

umm-spec-lib/resources/json-schemas/collection/umm/v1.17.3/umm-cmn-json-schema.json

@@ -518,6 +518,7 @@
     "DoiType": {
       "oneOf": [{
         "type": "object",
+        "title": "DOI is available",
         "additionalProperties": false,
         "description": "This element stores the DOI (Digital Object Identifier) that identifies the collection. Note: The values should start with the directory indicator which in ESDIS' case is 10.  If the DOI was registered through ESDIS, the beginning of the string should be 10.5067. The DOI URL is not stored here; it should be stored as a RelatedURL. The DOI organization that is responsible for creating the DOI is described in the Authority element. For ESDIS records the value of https://doi.org/ should be used. For those that want to specify that a DOI is not applicable or unknown for their record, use the second option.",
         "properties": {
@@ -535,6 +536,7 @@
         "required": ["DOI"]
       }, {
         "type": "object",
+        "title": "DOI is not available or applicable",
         "additionalProperties": false,
         "description": "This element stores the fact that the DOI (Digital Object Identifier) is not applicable or is unknown.",
         "properties": {

umm-spec-lib/src/cmr/umm_spec/test/iso19115_expected_conversion.clj

@@ -1,30 +1,23 @@
 (ns cmr.umm-spec.test.iso19115-expected-conversion
   "ISO 19115 specific expected conversion functionality"
   (:require
-    [clj-time.core :as t]
     [clj-time.format :as f]
     [clojure.string :as string]
     [cmr.common.util :as util :refer [update-in-each]]
     [cmr.spatial.mbr :as m]
     [cmr.umm-spec.date-util :as date-util]
     [cmr.umm-spec.iso19115-2-util :as iso-util]
     [cmr.umm-spec.json-schema :as js]
-    [cmr.umm-spec.location-keywords :as lk]
     [cmr.umm-spec.models.umm-collection-models :as umm-c]
     [cmr.umm-spec.models.umm-common-models :as cmn]
-    [cmr.umm-spec.related-url :as ru-gen]
     [cmr.umm-spec.spatial-conversion :as spatial-conversion]
     [cmr.umm-spec.test.expected-conversion-util :as conversion-util]
     [cmr.umm-spec.test.iso-shared :as iso-shared]
-    [cmr.umm-spec.test.location-keywords-helper :as lkt]
-    [cmr.umm-spec.umm-to-xml-mappings.iso19115-2 :as iso]
     [cmr.umm-spec.umm-to-xml-mappings.iso19115-2.additional-attribute :as iso-aa]
     [cmr.umm-spec.umm-to-xml-mappings.iso19115-2.data-contact :as data-contact]
     [cmr.umm-spec.url :as url]
     [cmr.umm-spec.util :as su]
-    [cmr.umm-spec.xml-to-umm-mappings.characteristics-data-type-normalization :as char-data-type-normalization]
-    [cmr.umm-spec.xml-to-umm-mappings.iso-shared.iso-topic-categories :as iso-topic-categories]
-    [cmr.umm-spec.xml-to-umm-mappings.iso19115-2.data-contact :as xml-to-umm-data-contact]))
+    [cmr.umm-spec.xml-to-umm-mappings.characteristics-data-type-normalization :as char-data-type-normalization]))
 
 (defn- propagate-first
   "Returns coll with the first element's value under k assoc'ed to each element in coll.
@@ -73,13 +66,6 @@
              (update :ISBN su/format-isbn)
              (update :OnlineResource expected-online-resource)))))
 
-(defn- expected-iso-19115-2-distributions
-  "Returns the expected ISO19115-2 distributions for comparison."
-  [distributions]
-  (some->> distributions
-           su/remove-empty-records
-           vec))
-
 (defn expected-iso-19115-2-related-urls
   [related-urls]
   (when (seq related-urls)
@@ -340,33 +326,33 @@
 
 (defn update-horizontal-data-resolutions
   "Add the type and remove nil keys in HorizontalDataResolution."
-  [c]
-  (if (seq (get-in c [:SpatialExtent :HorizontalSpatialDomain :ResolutionAndCoordinateSystem :HorizontalDataResolution]))
-    (-> c
-        (util/update-in-each
-          [:SpatialExtent :HorizontalSpatialDomain :ResolutionAndCoordinateSystem :HorizontalDataResolution :NonGriddedResolutions]
-          util/remove-nil-keys)
-        (util/update-in-each
-          [:SpatialExtent :HorizontalSpatialDomain :ResolutionAndCoordinateSystem :HorizontalDataResolution :NonGriddedRangeResolutions]
-          util/remove-nil-keys)
-        (util/update-in-each
-          [:SpatialExtent :HorizontalSpatialDomain :ResolutionAndCoordinateSystem :HorizontalDataResolution :GenericResolutions]
-          util/remove-nil-keys)
-        (util/update-in-each
-          [:SpatialExtent :HorizontalSpatialDomain :ResolutionAndCoordinateSystem :HorizontalDataResolution :GriddedResolutions]
-          util/remove-nil-keys)
-        (util/update-in-each
-          [:SpatialExtent :HorizontalSpatialDomain :ResolutionAndCoordinateSystem :HorizontalDataResolution :GriddedRangeResolutions]
-          util/remove-nil-keys)
-        ;; The order here is important because calling umm-c/map->HorizontalDataResolutionType will put back the nil values if
-        ;; remove-nil-keys is called first.
-        (update-in
-          [:SpatialExtent :HorizontalSpatialDomain :ResolutionAndCoordinateSystem :HorizontalDataResolution]
-          umm-c/map->HorizontalDataResolutionType)
-        (update-in
-          [:SpatialExtent :HorizontalSpatialDomain :ResolutionAndCoordinateSystem :HorizontalDataResolution]
-          util/remove-nil-keys))
-    c))
+  [horizontal-data-resolution]
+  (-> horizontal-data-resolution
+      (util/update-in-each [:NonGriddedResolutions] umm-c/map->HorizontalDataResolutionNonGriddedType)
+      (util/update-in-each [:NonGriddedRangeResolutions] umm-c/map->HorizontalDataResolutionNonGriddedRangeType)
+      (util/update-in-each [:GenericResolutions] umm-c/map->HorizontalDataGenericResolutionType)
+      (util/update-in-each [:GriddedResolutions] umm-c/map->HorizontalDataResolutionGriddedType)
+      (util/update-in-each [:GriddedRangeResolutions] umm-c/map->HorizontalDataResolutionGriddedRangeType)
+      umm-c/map->HorizontalDataResolutionType))
+
+(defn expected-horizontal-spatial-domain
+  [horizontal-spatial-domain]
+  (if-let [res (:ResolutionAndCoordinateSystem horizontal-spatial-domain)]
+    (as-> res res
+      (if (:Description res)
+        (update res :Description iso-util/safe-trim)
+        res)
+      (if (:GeodeticModel res)
+        (update res :GeodeticModel expected-geodetic-model)
+        res)
+      (if (:LocalCoordinateSystem res)
+        (update res :LocalCoordinateSystem expected-local-coordinate-system)
+        res)
+      (if (:HorizontalDataResolution res)
+        (update res :HorizontalDataResolution update-horizontal-data-resolutions)
+        res)
+      (assoc horizontal-spatial-domain :ResolutionAndCoordinateSystem res))
+    horizontal-spatial-domain))
 
 (defn umm-expected-conversion-iso19115
   [umm-coll]
@@ -408,8 +394,5 @@
       (update :DOI iso-shared/expected-doi)
       (update :UseConstraints iso-shared/expected-use-constraints)
       (update :ArchiveAndDistributionInformation iso-shared/expected-archive-dist-info)
-      (update-in [:SpatialExtent :HorizontalSpatialDomain :ResolutionAndCoordinateSystem :Description] iso-util/safe-trim)
-      (update-in [:SpatialExtent :HorizontalSpatialDomain :ResolutionAndCoordinateSystem :GeodeticModel] expected-geodetic-model)
-      (update-in [:SpatialExtent :HorizontalSpatialDomain :ResolutionAndCoordinateSystem :LocalCoordinateSystem] expected-local-coordinate-system)
-      (update-horizontal-data-resolutions)
+      (update-in [:SpatialExtent :HorizontalSpatialDomain] expected-horizontal-spatial-domain)
       js/parse-umm-c))

umm-spec-lib/src/cmr/umm_spec/test/umm_generators.clj

@@ -145,7 +145,7 @@
 (defmethod schema-type->generator "object"
   [schema type-name schema-type]
   (rejected-unexpected-fields #{:properties :additionalProperties :required :oneOf
-                                :anyOf :allOf :if :then :else :not :dependencies :$id} schema-type)
+                                :anyOf :allOf :if :then :else :not :dependencies :$id :title} schema-type)
   (if-let [one-of (:oneOf schema-type)]
     (object-one-of->generator schema type-name schema-type)
     ;; else

umm-spec-lib/src/cmr/umm_spec/umm_to_xml_mappings/iso19115_2/spatial.clj

@@ -111,18 +111,18 @@
            NumberOfOrbits StartCircularLatitude StartCircularLatitudeUnit]}]
   (let [main-string (format "OrbitPeriod: %s OrbitPeriodUnit: %s
                              InclinationAngle: %s InclinationAngleUnit: %s NumberOfOrbits: %s"
-                            (util/double->string OrbitPeriod)
+                            OrbitPeriod
                             OrbitPeriodUnit
-                            (util/double->string InclinationAngle)
+                            InclinationAngle
                             InclinationAngleUnit
-                            (util/double->string NumberOfOrbits))
+                            NumberOfOrbits)
         sw-string (when SwathWidth
                     (format "SwathWidth: %s SwathWidthUnit: %s "
-                            (util/double->string SwathWidth)
+                            SwathWidth
                             SwathWidthUnit))
         scl-string (when StartCircularLatitude
                      (format " StartCircularLatitude: %s StartCircularLatitudeUnit: %s"
-                             (util/double->string StartCircularLatitude)
+                             StartCircularLatitude
                              StartCircularLatitudeUnit))]
     (str sw-string main-string scl-string)))
 
@@ -198,7 +198,6 @@
                       (get-in c [:SpatialExtent :HorizontalSpatialDomain :ResolutionAndCoordinateSystem
                                  :HorizontalDataResolution]))
         resolutions (flatten (map #(package-horizontal-data-resolutions %) resolutions))]
-    (def resolutions resolutions)
     (for [[id res] (map-indexed vector resolutions)]
       (assoc res :Count id))))
 

umm-spec-lib/src/cmr/umm_spec/xml_to_umm_mappings/dif10/spatial.clj

@@ -115,42 +115,49 @@
         [geom] (select spatial-coverage "Geometry")
         horizontal-data-resolutions (parse-horizontal-data-resolutions doc)
         local-coordinate-sys (parse-local-coord-system spatial-coverage)
-        geodetic-model (parse-geodetic-model spatial-coverage)]
+        geodetic-model (parse-geodetic-model spatial-coverage)
+        resolution (if (or geodetic-model
+                           local-coordinate-sys
+                           horizontal-data-resolutions)
+                     {:ResolutionAndCoordinateSystem {:GeodeticModel geodetic-model
+                                                      :LocalCoordinateSystem local-coordinate-sys
+                                                      :HorizontalDataResolution horizontal-data-resolutions}}
+                     {:ResolutionAndCoordinateSystem nil})]
     (util/remove-nil-keys
+     (merge
       {:Geometry (parse-geometry geom)
-       :ZoneIdentifier (value-of spatial-coverage "Zone_Identifier")
-       :ResolutionAndCoordinateSystem {:GeodeticModel geodetic-model
-                                       :LocalCoordinateSystem local-coordinate-sys
-                                       :HorizontalDataResolution horizontal-data-resolutions}})))
+       :ZoneIdentifier (value-of spatial-coverage "Zone_Identifier")}
+       resolution))))
 
 (defn parse-spatial
   "Returns UMM-C spatial map from DIF 10 XML document."
   [doc]
-  (let [[spatial] (select doc "/DIF/Spatial_Coverage")]
-    {:SpatialCoverageType (dif10-spatial-type->umm-spatial-type (value-of spatial "Spatial_Coverage_Type"))
-     :GranuleSpatialRepresentation (value-of spatial "Granule_Spatial_Representation")
-     :HorizontalSpatialDomain      (parse-horizontal-spatial-domains doc)
-     :VerticalSpatialDomains       (spatial-conversion/convert-vertical-spatial-domains-from-xml
-                                    (select spatial "Vertical_Spatial_Info"))
-     :OrbitParameters             (let [[o] (select spatial "Orbit_Parameters")]
-                                     (as->{:SwathWidth (value-of o "Swath_Width")
-                                           :OrbitPeriod (value-of o "Period")
-                                           :InclinationAngle (value-of o "Inclination_Angle")
-                                           :NumberOfOrbits (value-of o "Number_Of_Orbits")
-                                           :StartCircularLatitude (value-of o "Start_Circular_Latitude")} op
-                                           ;; Add assumed units for the corresponding fields.
-                                           (if (:SwathWidth op)
-                                             (assoc op :SwathWidthUnit "Kilometer")
-                                             op)
-                                           (if (:OrbitPeriod op)
-                                             (assoc op :OrbitPeriodUnit "Decimal Minute")
-                                             op)
-                                           (if (:InclinationAngle op)
-                                             (assoc op :InclinationAngleUnit "Degree")
-                                             op)
-                                           (if (:StartCircularLatitude op)
-                                             (assoc op :StartCircularLatitudeUnit "Degree")
-                                             op)))}))
+  (let [[spatial] (select doc "/DIF/Spatial_Coverage")
+        [orbit-parameters] (select spatial "Orbit_Parameters")]
+      {:SpatialCoverageType (dif10-spatial-type->umm-spatial-type (value-of spatial "Spatial_Coverage_Type"))
+       :GranuleSpatialRepresentation (value-of spatial "Granule_Spatial_Representation")
+       :HorizontalSpatialDomain      (parse-horizontal-spatial-domains doc)
+       :VerticalSpatialDomains       (spatial-conversion/convert-vertical-spatial-domains-from-xml
+                                      (select spatial "Vertical_Spatial_Info"))
+       :OrbitParameters              (when orbit-parameters
+                                       (as->{:SwathWidth (util/safe-read-string (value-of orbit-parameters "Swath_Width"))
+                                             :OrbitPeriod (util/safe-read-string (value-of orbit-parameters "Period"))
+                                             :InclinationAngle (util/safe-read-string (value-of orbit-parameters "Inclination_Angle"))
+                                             :NumberOfOrbits (util/safe-read-string (value-of orbit-parameters "Number_Of_Orbits"))
+                                             :StartCircularLatitude (util/safe-read-string (value-of orbit-parameters "Start_Circular_Latitude"))} op
+                                             ;; Add assumed units for the corresponding fields.
+                                             (if (:SwathWidth op)
+                                               (assoc op :SwathWidthUnit "Kilometer")
+                                               op)
+                                             (if (:OrbitPeriod op)
+                                               (assoc op :OrbitPeriodUnit "Decimal Minute")
+                                               op)
+                                             (if (:InclinationAngle op)
+                                               (assoc op :InclinationAngleUnit "Degree")
+                                               op)
+                                             (if (:StartCircularLatitude op)
+                                               (assoc op :StartCircularLatitudeUnit "Degree")
+                                               op)))}))
 
 (def tiling-system-xpath
   "/DIF/Spatial_Coverage/Spatial_Info/TwoD_Coordinate_System")

umm-spec-lib/src/cmr/umm_spec/xml_to_umm_mappings/echo10/spatial.clj

@@ -89,14 +89,20 @@
         horizontal-data-resolutions (parse-horizontal-data-resolutions spatial-info)
         local-coordinate-sys (parse-local-coord-system spatial-info)
         geodetic-model (parse-geodetic-model spatial-info)
+        resolution (if (or geodetic-model
+                           local-coordinate-sys
+                           horizontal-data-resolutions)
+                     {:ResolutionAndCoordinateSystem {:GeodeticModel geodetic-model
+                                                      :LocalCoordinateSystem local-coordinate-sys
+                                                      :HorizontalDataResolution horizontal-data-resolutions}}
+                     {:ResolutionAndCoordinateSystem nil})
         [horiz] (select doc "/Collection/Spatial/HorizontalSpatialDomain")]
     (umm-c/map->HorizontalSpatialDomainType
      (util/remove-nil-keys
-      {:Geometry (parse-geometry (first (select horiz "Geometry")))
-       :ZoneIdentifier (value-of horiz "ZoneIdentifier")
-       :ResolutionAndCoordinateSystem {:GeodeticModel geodetic-model
-                                       :LocalCoordinateSystem local-coordinate-sys
-                                       :HorizontalDataResolution horizontal-data-resolutions}}))))
+      (merge
+       {:Geometry (parse-geometry (first (select horiz "Geometry")))
+        :ZoneIdentifier (value-of horiz "ZoneIdentifier")}
+       resolution)))))
 
 (defn parse-spatial
   "Returns UMM-C spatial map from ECHO10 XML document."
@@ -108,27 +114,25 @@
      :HorizontalSpatialDomain      (parse-horizontal-spatial-domain doc)
      :VerticalSpatialDomains       (spatial-conversion/convert-vertical-spatial-domains-from-xml
                                     (select spatial "VerticalSpatialDomain"))
-     :OrbitParameters              (as->(fields-from (first (select spatial "OrbitParameters"))
-                                                     :SwathWidth
-                                                     :Period
-                                                     :InclinationAngle
-                                                     :NumberOfOrbits
-                                                     :StartCircularLatitude) op
-                                        ;; Add assumed units for the corresponding fields.
-                                        ;; Replace :Period with :OrbitPeriod.
-                                        (if (:SwathWidth op) 
-                                          (assoc op :SwathWidthUnit "Kilometer")
-                                          op)
-                                        (if (:Period op) 
-                                          (assoc op :OrbitPeriod (:Period op)
-                                                    :OrbitPeriodUnit "Decimal Minute")
-                                          op)
-                                        (if (:InclinationAngle op)
-                                          (assoc op :InclinationAngleUnit "Degree")
-                                          op)
-                                        (if (:StartCircularLatitude op)
-                                          (assoc op :StartCircularLatitudeUnit "Degree")
-                                          op)
-                                        (dissoc op :Period))}
+     :OrbitParameters              (when-let [[orbit-parameters] (select spatial "OrbitParameters")]
+                                     (as-> {:SwathWidth (util/safe-read-string (value-of orbit-parameters "SwathWidth"))
+                                            :OrbitPeriod (util/safe-read-string (value-of orbit-parameters "Period"))
+                                            :InclinationAngle (util/safe-read-string (value-of orbit-parameters "InclinationAngle"))
+                                            :NumberOfOrbits (util/safe-read-string (value-of orbit-parameters "NumberOfOrbits"))
+                                            :StartCircularLatitude (util/safe-read-string (value-of orbit-parameters "StartCircularLatitude"))} op
+                                         ;; Add assumed units for the corresponding fields.
+                                         ;; Replace :Period with :OrbitPeriod.
+                                         (if (:SwathWidth op)
+                                           (assoc op :SwathWidthUnit "Kilometer")
+                                           op)
+                                         (if (:OrbitPeriod op)
+                                           (assoc op :OrbitPeriodUnit "Decimal Minute")
+                                           op)
+                                         (if (:InclinationAngle op)
+                                           (assoc op :InclinationAngleUnit "Degree")
+                                           op)
+                                         (if (:StartCircularLatitude op)
+                                           (assoc op :StartCircularLatitudeUnit "Degree")
+                                           op)))}
     (when sanitize?
       u/not-provided-spatial-extent)))