CMR-10173: Update UMM-C for Web Unitification round 2

common-app-lib/src/cmr/common_app/config.clj

@@ -8,7 +8,7 @@
 (defconfig collection-umm-version
   "Defines the latest collection umm version accepted by ingest - it's the latest official version.
    This environment variable needs to be manually set when newer UMM version becomes official"
-  {:default "1.18.1"})
+  {:default "1.18.2"})
 
 (defconfig launchpad-token-enforced
   "Flag for whether or not launchpad token is enforced."

umm-spec-lib/resources/json-schemas/collection/umm/v1.18.2/umm-search-results-json-schema.json

@@ -0,0 +1,227 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "definitions": {
+    "ItemType": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "Represents a single item found in search results. It contains some metadata about the item found along with the UMM representing the item. umm won't be present if the item represents a tombstone.",
+      "properties": {
+        "meta": {
+          "$ref": "#/definitions/MetaType"
+        },
+        "umm": {
+          "$ref": "umm-c-json-schema.json"
+        }
+      },
+      "required": ["meta"]
+    },
+    "MetaType": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "CMR level metadata about the item found. This represents data not found in the actual metadata.",
+      "properties": {
+        "granule-count": {
+          "description": "A number >= 0 that indicates the number of granules for the collection.",
+          "type": "number"
+        },
+        "provider-id": {
+          "description": "The identity of the provider in the CMR",
+          "type": "string",
+          "minLength": 1,
+          "pattern": "[A-Z0-9_]+"
+        },
+        "concept-type": {
+          "description": "The type of item found.",
+          "type": "string",
+          "enum": ["collection"]
+        },
+        "native-id": {
+          "description": "The id used by the provider to identify this item during ingest.",
+          "type": "string",
+          "minLength": 1
+        },
+        "concept-id": {
+          "description": "The concept id of the item found.",
+          "$ref": "#/definitions/ConceptIdType"
+        },
+        "revision-id": {
+          "description": "A number >= 1 that indicates which revision of the item.",
+          "type": "number"
+        },
+        "revision-date": {
+          "description": "The date this revision was created. This would be the creation or update date of the item in the CMR.",
+          "format": "date-time",
+          "type": "string"
+        },
+        "user-id": {
+          "description": "The id of the user who created this item revision.",
+          "type": "string",
+          "minLength": 1
+        },
+        "deleted": {
+          "description": "Indicates if the item represents a tombstone",
+          "type": "boolean"
+        },
+        "format": {
+          "description": "The mime type of the original metadata",
+          "type": "string",
+          "minLength": 1
+        },
+        "has-variables": {
+          "description": "Indicates if there are variables associated with the collection",
+          "type": "boolean"
+        },
+        "has-formats": {
+          "description": "Indicates if there are multiple supported formats for any services associated with the collection",
+          "type": "boolean"
+        },
+        "has-transforms": {
+          "description": "Indicates if there are transformations (subset, interpolation or projection) in any services associated with the collection",
+          "type": "boolean"
+        },
+        "has-combine": {
+          "description": "Indicates if there is concatenation available in any services associated with the collection",
+          "type": "boolean"
+        },
+        "has-spatial-subsetting": {
+          "description": "Indicates if any associated services support spatial subsetting",
+          "type": "boolean"
+        },
+        "has-temporal-subsetting": {
+          "description": "Indicates if any associated services support temporal subsetting",
+          "type": "boolean"
+        },
+        "associations": {
+          "description": "Contains variables and services associated with the collection",
+          "$ref": "#/definitions/AssociationsType"
+        },
+        "association-details": {
+          "description": "Contains the details of each association.",
+          "$ref": "#/definitions/AssociationDetailsType"
+        },
+        "s3-links": {
+          "description": "Contains the DirectDistributionInformation S3 links associated with the collection.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      },
+      "required": ["provider-id", "concept-type", "native-id", "concept-id", "revision-id", "revision-date"]
+    },
+    "AssociationsType": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "Any concept associated with the collection.",
+      "properties": {
+        "variables": {
+          "description": "The concept ids of variables associated with the collection",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        },
+        "services": {
+          "description": "The concept ids of services associated with the collection",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        },
+        "tools": {
+          "description": "The concept ids of tools associated with the collection",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        },
+        "data-quality-summaries": {
+          "description": "The concept ids of data quality summary concepts associated with the collection",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        },
+        "order-options": {
+          "description": "The concept ids of order option concepts associated with the collection",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        },
+        "service-options": {
+          "description": "The concept ids of service option concepts associated with the collection",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        },
+        "service-entries": {
+          "description": "The concept ids of service entry concepts associated with the collection",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        },
+        "grids": {
+          "description": "The concept ids of grid concepts associated with the collection",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ConceptIdType"
+          },
+          "minItems": 1
+        }
+      }
+    },
+    "AssociationDetailsType": {
+      "type": "object",
+      "additionalProperties": true,
+      "description": "Association details of concepts associated with the collection.",
+      "properties": {
+        "concept-id": {
+          "description": "The concept id of the associated concept.",
+          "type": "string"
+        },
+        "revision-id": {
+          "description": "The revision id of the associated concept.",
+          "type": "string"
+        }
+      }
+    },
+    "ConceptIdType": {
+      "description": "The concept id of a concept.",
+      "type": "string",
+      "minLength": 4,
+      "pattern": "[A-Z]+\\d+-[A-Z0-9_]+"
+    }
+  },
+  "title": "UMM Search Results",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "hits": {
+      "description": "The total number of items that matched the search.",
+      "type": "number"
+    },
+    "took": {
+      "description": "How long the search took in milliseconds from the time the CMR received the request until it had generated the response. This does not include network traffic time to send the request or return the response.",
+      "type": "number"
+    },
+    "items": {
+      "description": "The list of items matching the result in this page.",
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/ItemType"
+      },
+      "minItems": 0
+    }
+  },
+  "required": ["hits", "took", "items"]
+}

umm-spec-lib/resources/xml-schemas/dif10/UmmCommon_1.2.xsd

@@ -580,6 +580,8 @@ technical contact
             <xs:enumeration value="Parent Dataset"/>
             <xs:enumeration value="Related Dataset"/>
             <xs:enumeration value="Other"/>
+            <xs:enumeration value="IsPreviousVersionOf"/>
+            <xs:enumeration value="IsNewVersionOf"/>
         </xs:restriction>
     </xs:simpleType>
     <xs:simpleType name="AssociatedDOITypeOtherEnum">

umm-spec-lib/resources/xml-schemas/echo10/MetadataCommon.xsd

@@ -817,6 +817,8 @@
 						<xs:enumeration value="Parent Dataset"/>
 						<xs:enumeration value="Related Dataset"/>
 						<xs:enumeration value="Other"/>
+                        <xs:enumeration value="IsPreviousVersionOf"/>
+                        <xs:enumeration value="IsNewVersionOf"/>
 					</xs:restriction>
 				</xs:simpleType>
 			</xs:element>

umm-spec-lib/src/cmr/umm_spec/migration/version/collection.clj

@@ -740,4 +740,40 @@
   [_context collection & _]
   ;; File sizes from positive numbers to numbers, only need to migrate version. 
   (-> collection
-      (m-spec/update-version :collection "1.18.0"))) 
+      (m-spec/update-version :collection "1.18.0")))
+
+(defmethod interface/migrate-umm-version [:collection "1.18.1" "1.18.2"]
+   [_context collection & _]
+   ;; Migrating up version 1.18.1 to 1.18.2
+   ;; Add AssociatedDOIs/Type enums: IsPreviousVersionOf and IsNewVersionOf
+   ;; Add PREPRINT, INREVIEW, and SUPERSEDED enums to CollectionProgress
+   ;; Remove NOT APPLICABLE enum from CollectionProgress
+   (m-spec/update-version collection :collection "1.18.2")
+   ;; Change CollectionProgress to "NOT PROVIDED" if its value is "NOT APPLICABLE"
+   (let [CollectionProgress (:CollectionProgress collection)]
+        (if (or (= "NOT APPLICABLE" CollectionProgress))
+          (assoc collection :CollectionProgress "NOT PROVIDED")
+          collection)))
+
+(defmethod interface/migrate-umm-version [:collection "1.18.2" "1.18.1"]
+           [_context collection & _]
+   ;; Migrating down version 1.18.2 to 1.18.1
+   ;; Remove AssociatedDOIs/Type enums: IsPreviousVersionOf and IsNewVersionOf
+   ;; Remove PREPRINT, INREVIEW, and SUPERSEDED enums to CollectionProgress
+   ;; Add back in NOT APPLICABLE enum in CollectionProgress
+   (m-spec/update-version collection :collection "1.18.1")
+   ;; Change AssociatedDOIs/Type to 'Related Dataset' if its enum value is IsPreviousVersionOf and IsNewVersionOf
+   ;; TODO
+   ;(as-> rc
+   ;      (let [sct (get-in rc [:AssociatedDOIs :Type])]
+   ;           (if (or (= "IsPreviousVersionOf" sct)
+   ;                   (= "IsNewVersionOf" sct))
+   ;             (update-in rc [:AssociatedDOIs] dissoc :Type)
+   ;             rc)))
+   ;; Change CollectionProgress enum to COMPLETED if its enum value is PREPRINT, INREVIEW, or SUPERSEDED
+   (let [CollectionProgress (:CollectionProgress collection)]
+        (if (or (= "PREPRINT" CollectionProgress)
+                (= "INREVIEW" CollectionProgress)
+                (= "SUPERSEDED" CollectionProgress))
+          (assoc collection :CollectionProgress "COMPLETED")
+          collection)))

umm-spec-lib/src/cmr/umm_spec/models/umm_collection_models.clj

@@ -151,17 +151,21 @@
    ;; The title of the collection or service described by the metadata.
    EntryTitle
 
-   ;; This element describes the production status of the data set. There are five choices for Data
+   ;; This element describes the production status of the data set. There are multiple choices for Data
    ;; Providers: PLANNED refers to data sets to be collected in the future and are thus unavailable
    ;; at the present time. For Example: The Hydro spacecraft has not been launched, but information
    ;; on planned data sets may be available. ACTIVE refers to data sets currently in production or
    ;; data that is continuously being collected or updated. For Example: data from the AIRS
    ;; instrument on Aqua is being collected continuously. COMPLETE refers to data sets in which no
    ;; updates or further data collection will be made. For Example: Nimbus-7 SMMR data collection
    ;; has been completed. DEPRECATED refers to data sets that have been retired, but still can be
-   ;; retrieved. Usually newer products exist that replace the retired data set. NOT APPLICABLE
-   ;; refers to data sets in which a collection progress is not applicable such as a calibration
-   ;; collection. There is a sixth value of NOT PROVIDED that should not be used by a data provider.
+   ;; retrieved. Usually newer products exist that replace the retired data set.
+   ;; PREPRINT: Refers to datasets which are made available prior to completion of validation and review processes to support manuscript publication processes and open science.
+   ;; Preprint datasets are provisional and should not be used for production applications.
+   ;; INREVIEW: Refers to datasets which are made available to support science team final review. In Review datasets are provisional and should not be used for production applications.
+   ;; Note that if restricted access is needed, an INREVIEW dataset may also have an Access Control List applied.
+   ;; SUPERSEDED: Refers to datasets which remain publicly available, but for which a newer version is available.
+   ;; There is a sixth value of NOT PROVIDED that should not be used by a data provider.
    ;; It is currently being used as a value when a correct translation cannot be done with the
    ;; current valid values, or when the value is not provided by the data provider.
    CollectionProgress

umm-spec-lib/src/cmr/umm_spec/umm_to_xml_mappings/dif10.clj

@@ -13,12 +13,18 @@
     [cmr.umm-spec.umm-to-xml-mappings.dif10.spatial :as spatial]
     [cmr.umm-spec.util :as u]))
 
+;; TODO?
 (def coll-progress-mapping
   "Mapping from known collection progress values to values supported for DIF10 Dataset_Progress."
   {"COMPLETE" "COMPLETE"
    "ACTIVE" "IN WORK"
    "PLANNED" "PLANNED"
-   "DEPRECATED" "COMPLETE"})
+   "DEPRECATED" "COMPLETE"
+   "NOT PROVIDED" "NOT PROVIDED"
+   "PREPRINT" "PREPRINT"
+   "INREVIEW" "INREVIEW"
+   "SUPERSEDED" "SUPERSEDED"
+   })
 
 (def platform-types
   "The set of values that DIF 10 defines for platform types as enumerations in its schema"

umm-spec-lib/src/cmr/umm_spec/umm_to_xml_mappings/iso_shared/collection_progress.clj

@@ -11,7 +11,10 @@
    "ACTIVE" "onGoing"
    "PLANNED" "planned"
    "DEPRECATED" "deprecated"
-   "NOT APPLICABLE" "NOT APPLICABLE"})
+   "NOT PROVIDED" "notprovided"
+   "PREPRINT" "PREPRINT"
+   "INREVIEW" "INREVIEW"
+   "SUPERSEDED" "SUPERSEDED"})
 
 (defn generate-collection-progress
   "Returns ISO CollectionProgress element from UMM-C collection c."

umm-spec-lib/src/cmr/umm_spec/umm_to_xml_mappings/iso_shared/doi.clj

@@ -58,7 +58,9 @@
    "Field Campaign" ["fieldCampaign" "Field_Campaign"]
    "Parent Dataset" ["parentDataset" "Parent_Dataset"]
    "Related Dataset" ["relatedDataset" "Related_Dataset"]
-   "Other" ["other" "Other"]})
+   "Other" ["other" "Other"]
+   "IsPreviousVersionOf" ["isPreviousVersionOf" "Is_Previous_Version_Of"]
+   "IsNewVersionOf" ["isNewVersionOf" "Is_New_Version_Of"]})
 
 (defn generate-associated-dois
   "Generate from a UMM-C record the AssociatedDOIs into the ISO MENDS/SMAP form."

umm-spec-lib/src/cmr/umm_spec/versioning.clj

@@ -13,7 +13,7 @@
                 "1.15" "1.15.1" "1.15.2" "1.15.3" "1.15.4" "1.15.5"
                 "1.16" "1.16.1" "1.16.2" "1.16.3" "1.16.4" "1.16.5" "1.16.6" "1.16.7"
                 "1.17.0" "1.17.1" "1.17.2" "1.17.3"
-                "1.18.0" "1.18.1"]
+                "1.18.0" "1.18.1" "1.18.2"]
    :granule ["1.4" "1.5" "1.6" "1.6.1" "1.6.2" "1.6.3" "1.6.4" "1.6.5" "1.6.6"]
    :variable ["1.0" "1.1" "1.2" "1.3" "1.4" "1.5" "1.6" "1.7" "1.8" "1.8.1" "1.8.2" "1.9.0"]
    :service ["1.0" "1.1" "1.2" "1.3" "1.3.1" "1.3.2" "1.3.3" "1.3.4" "1.4" "1.4.1" "1.5.0" "1.5.1" "1.5.2" "1.5.3"]

umm-spec-lib/src/cmr/umm_spec/xml_to_umm_mappings/dif10.clj

@@ -24,6 +24,7 @@
    [cmr.umm-spec.versioning :as umm-spec-versioning]
    [cmr.umm.dif.date-util :refer [parse-dif-end-date]]))
 
+;; TODO how do we know which values are supported for this version?
 (def coll-progress-mapping
   "Mapping from values supported for DIF10 Data_Set_Progress to UMM CollectionProgress."
   {"COMPLETE" "COMPLETE"

umm-spec-lib/src/cmr/umm_spec/xml_to_umm_mappings/dif9.clj

@@ -19,6 +19,7 @@
    [cmr.umm-spec.versioning :as umm-spec-versioning]
    [cmr.umm.dif.date-util :refer [parse-dif-end-date]]))
 
+;; TODO
 (def coll-progress-mapping
   "Mapping from values supported for DIF9 Data_Set_Progress to UMM CollectionProgress."
   {"COMPLETE" "COMPLETE"

umm-spec-lib/src/cmr/umm_spec/xml_to_umm_mappings/echo10.clj

@@ -22,6 +22,7 @@
   (:import
    (clojure.data.xml Element)))
 
+;; TODO
 (def coll-progress-mapping
   "Mapping from values supported for ECHO10 CollectionState to UMM CollectionProgress."
   {"COMPLETE" "COMPLETE"