From a08377444acb0db6ff2fa2b58150669577982242 Mon Sep 17 00:00:00 2001 From: Erich Reiter Date: Wed, 23 Aug 2023 15:07:00 -0400 Subject: [PATCH 1/3] CMR-9392 adding Grid Draft records to the CMR. --- common-lib/src/cmr/common/config.clj | 3 +- .../src/cmr/ingest/api/generic_documents.clj | 7 +- .../schemas/grid-draft/v1.0.0/index.json | 28 +++ .../schemas/grid-draft/v1.0.0/ingest.md | 85 +++++++ .../schemas/grid-draft/v1.0.0/metadata.json | 118 ++++++++++ .../schemas/grid-draft/v1.0.0/search.md | 218 ++++++++++++++++++ 6 files changed, 454 insertions(+), 5 deletions(-) create mode 100644 schemas/resources/schemas/grid-draft/v1.0.0/index.json create mode 100644 schemas/resources/schemas/grid-draft/v1.0.0/ingest.md create mode 100644 schemas/resources/schemas/grid-draft/v1.0.0/metadata.json create mode 100644 schemas/resources/schemas/grid-draft/v1.0.0/search.md diff --git a/common-lib/src/cmr/common/config.clj b/common-lib/src/cmr/common/config.clj index f6c77b77d7..91340e9466 100644 --- a/common-lib/src/cmr/common/config.clj +++ b/common-lib/src/cmr/common/config.clj @@ -233,7 +233,8 @@ :service-draft ["1.0.0"] :variable-draft ["1.0.0"] :data-quality-summary-draft ["1.0.0"] - :order-option-draft ["1.0.0"]} + :order-option-draft ["1.0.0"] + :grid-draft ["1.0.0"]} :parser #(json/parse-string % true)}) (defconfig approved-pipeline-documentation diff --git a/ingest-app/src/cmr/ingest/api/generic_documents.clj b/ingest-app/src/cmr/ingest/api/generic_documents.clj index 396dfe3a72..43b2ea68dd 100644 --- a/ingest-app/src/cmr/ingest/api/generic_documents.clj +++ b/ingest-app/src/cmr/ingest/api/generic_documents.clj @@ -56,9 +56,7 @@ {:collection-draft :umm-c :service-draft :umm-s :tool-draft :umm-t - :variable-draft :umm-var - :data-quality-summary-draft :data-quality-summary - :order-option-draft :order-option}) + :variable-draft :umm-var}) (defn is-draft-concept? "This function checks to see if the concept to be ingested or updated @@ -117,7 +115,8 @@ {:keys [spec-key spec-version document-name format]} (pull-metadata-specific-information request-context concept-type content-type raw-document)] (if (and (not= concept-type spec-key) - (not= (concept-type draft-concept->spec-map) spec-key)) + (not= (concept-type draft-concept->spec-map) spec-key) + (not= (common-concepts/get-concept-type-of-draft concept-type) spec-key)) (throw (UnsupportedOperationException. (format "%s version %s is not supported." spec-key spec-version))) {:concept (assoc {} :metadata raw-document diff --git a/schemas/resources/schemas/grid-draft/v1.0.0/index.json b/schemas/resources/schemas/grid-draft/v1.0.0/index.json new file mode 100644 index 0000000000..91da99191d --- /dev/null +++ b/schemas/resources/schemas/grid-draft/v1.0.0/index.json @@ -0,0 +1,28 @@ +{ + "MetadataSpecification": { + "URL": "https://cdn.earthdata.nasa.gov/generic/index/v0.0.1", + "Name": "Generic-Index", + "Version": "0.0.1" + }, + "Generic": { + "Name": "Grid Draft", + "Version": "1.0.0" + }, + "SubConceptType": "GD", + "IndexSetup" : { + "index": { + "number_of_shards": 3, + "number_of_replicas": 1, + "refresh_interval": "1s" + } + }, + "Indexes": + [ + { + "Description": "Draft Grid Name", + "Field": ".Name", + "Name": "name", + "Mapping": "string" + } + ] +} diff --git a/schemas/resources/schemas/grid-draft/v1.0.0/ingest.md b/schemas/resources/schemas/grid-draft/v1.0.0/ingest.md new file mode 100644 index 0000000000..664f80d38a --- /dev/null +++ b/schemas/resources/schemas/grid-draft/v1.0.0/ingest.md @@ -0,0 +1,85 @@ +## Grid Draft + +#### /providers/<provider-id>/grid-drafts/<native-id> + +### Create / Update a Grid Draft + +Grid Draft concepts can be created or updated by sending an HTTP PUT with the metadata to the URL `%CMR-ENDPOINT%/providers//grid-drafts/`. The response will include the [concept id](#concept-id) and the [revision id](#revision-id). The contents of the metadata is passed in the body of the request. + +``` + curl -XPOST \ + -H "Cmr-Pretty: true" \ + -H "Content-Type:application/vnd.nasa.cmr.umm+json" \ + -H "Authorization: Bearer XXXX" \ + "%CMR-ENDPOINT%/providers/PROV1/grid-drafts/sampleNativeId" \ + -d @samplGrid.json +``` + +#### Successful Response in XML + +``` + + + GD1200000000-PROV1 + 1 + + + +``` +Subsequent ingests to the Grid Draft record will result in updates to it's metadata as well as increment the revision-id of the record. + +#### Successful Response in JSON + +By passing the option `-H "Accept: application/json"` to `curl`, one may +get a JSON response: + +``` + {"concept-id":"GD1200000000-PROV1","revision-id":1,"warnings":null,"existing-errors":null} +``` + +### Delete a Grid Draft + +Grid Draft metadata can be deleted by sending an HTTP DELETE to the URL `%CMR-ENDPOINT%/providers//grid-drafts/`. The response will include the [concept id](#concept-id) and the [revision id](#revision-id) of the tombstone. + +``` + curl -XDELETE \ + -H "Cmr-Pretty: true" \ + -H "Authorization: Bearer XXXX" \ + %CMR-ENDPOINT%/providers/PROV1/grid-drafts/sampleNative23Id" +``` + +#### Successful Response in XML + +``` + + + GD1200000000-PROV1 + 2 + +``` + +#### Successful Response in JSON + +``` + {"concept-id":"GD1200000000-PROV1","revision-id":2,"warnings":null,"existing-errors":null} +``` + +Attempting to delete an already deleted record will return +the following error message + +#### Unsuccessful Response in XML + +``` + + + Concept with native-id [sampleNative23Id] and concept-id [GD1200000000-PROV1] is already deleted. + +``` + +#### Unsuccessful Response in JSON + +``` + "errors": [ + "Concept with native-id [sampleNative23Id] and concept-id [GD1200000000-PROV1] is already deleted." + ] +``` \ No newline at end of file diff --git a/schemas/resources/schemas/grid-draft/v1.0.0/metadata.json b/schemas/resources/schemas/grid-draft/v1.0.0/metadata.json new file mode 100644 index 0000000000..59c6c15396 --- /dev/null +++ b/schemas/resources/schemas/grid-draft/v1.0.0/metadata.json @@ -0,0 +1,118 @@ +{ + "MetadataSpecification": { + "URL": "https://cdn.earthdata.nasa.gov/generic/grid/v0.0.1", + "Name": "Grid", + "Version": "0.0.1" + }, + "Name": "Grid-A7-v1", + "LongName": "Grid A-7 version 1.0", + "Version": "v1.0", + "Description": "A sample grid", + "GridDefinition": { + "CoordinateReferenceSystemID": { + "Type": "EPSG", + "Code": "EPSG:4326", + "Title": "WGS84 - World Geodetic System 1984, used in GPS - EPSG:4326", + "URL": "https://epsg.io/4326" + }, + "DimensionSize": { + "Height": 1, + "Width": 1, + "Time": 2, + "Other": { + "Name": "Other Dimension Size", + "Value": "42", + "Description": "Details about the other dimension size." + } + }, + + "SpatialConstraint": { + "0_360_DegreeProjection": true, + "NorthBoundingCoordinate": 90.0, + "EastBoundingCoordinate": 0.0, + "SouthBoundingCoordinate": -90.0, + "WestBoundingCoordinate": 90.0, + "SpatialResolution": { + "Unit": "Degree", + "LongitudeResolution": "0.0005", + "LatitudeResolution": "90.000" + } + }, + "DimensionScale": { + "0_360_DegreeProjection": true, + "X": + { + "Minimum": "-360-00-00.000", + "Maximum": "360-00-00.00", + "Unit": "Degrees-Minutes-Seconds", + "Resolution": 0.125 + }, + "Y": + { + "Minimum": "-360-00-00.000", + "Maximum": "360-00-00.00", + "Unit": "Degrees-Minutes-Seconds", + "Resolution": 0.5 + }, + "Time": + { + "Minimum": "2020-01-01T00:00:00.001Z", + "Maximum": "2020-01-30T23:59:59Z", + "Unit": "DateTime", + "Resolution": 0.125 + }, + "Other": + { + "Minimum": 0, + "Maximum": 44, + "Unit": "Meter", + "Resolution": 0.5 + } + }, + + "Distortion": { + "Description": "Distortion around the grid edge", + "Percent": 31 + }, + "Uniform-Grid": true, + "Bounded-Grid": true + }, + "Organization": { + "ShortName": "NASA/GSFC/SED/ESD/GCDC/GESDISC", + "LongName": "Goddard Earth Sciences Data and Information Services Center (formerly Goddard DAAC), Global Change Data Center, Earth Sciences Division, Science and Exploration Directorate, Goddard Space Flight Center, NASA", + "RelatedURLs": [ + { + "URL": "https://example.gov", + "URLContentType": "C-Type", + "Type": "Type" + } + ], + "ContactMechanisms": [ + { + "Type": "Email", + "Value": "who@example.gov" + },{ + "Type": "Email", + "Value": "you@example.gov" + } + ] + }, + "MetadataDate": {"Create": "2022-04-20T08:00:00Z"}, + "RelatedURLs": [ + { + "URL": "https://example.gov", + "URLContentType": "C-Type", + "Type": "Type" + },{ + "URL": "https://example.gov/two", + "Description": "Details about the URL or page", + "URLContentType": "C-Type", + "Type": "Type" + } + ], + "AdditionalAttribute": { + "Name": "attribute-1", + "Description": "Sample", + "DataType": "STRING" + } +} \ No newline at end of file diff --git a/schemas/resources/schemas/grid-draft/v1.0.0/search.md b/schemas/resources/schemas/grid-draft/v1.0.0/search.md new file mode 100644 index 0000000000..66ce43eb82 --- /dev/null +++ b/schemas/resources/schemas/grid-draft/v1.0.0/search.md @@ -0,0 +1,218 @@ +### Grid Draft + +Grid Drafts are draft records that inform users about the types of Grids that are available when reprojecting data using services. Grid metadata is stored in the JSON format [UMM-Grid Schema](https://git.earthdata.nasa.gov/projects/EMFD/repos/otherschemas/browse/grid). + +#### Searching for Grid Drafts + +Grid Drafts can be searched for by sending a request to `%CMR-ENDPOINT%/grid-drafts`. XML reference, JSON and UMM JSON response formats are supported for Grid Draft searches. + +Grid Draft search results are paged. See [Paging Details](#paging-details) for more information on how to page through Grid Draft search results. + +##### Grid Draft Search Parameters + +The following parameters are supported when searching for Grid Drafts. + +##### Standard Parameters + +* page\_size +* page\_num +* pretty + +##### Grid Draft Matching Parameters + +These parameters will match fields within a Grid Draft. They are case insensitive by default. They support options specified. They also support searching with multiple values in the style of `name[]=key1&name[]=key2`. The values are bitwise `OR`ed together. + +* name - options: pattern, ignore\_case +* provider - options: pattern, ignore\_case +* native\_id - options: pattern, ignore\_case +* concept\_id + +``` + curl "%CMR-ENDPOINT%/grid-drafts?concept_id=GD1200000000-PROV1" +``` + +##### Grid Draft Search Response + +##### XML Reference + +The XML reference response format is used for returning references to search results. It consists of the following fields: + +| Field | Description | +| ---------- | -------------------------------------------------- | +| hits | the number of results matching the search query | +| took | time in milliseconds it took to perform the search | +| references | identifying information about each search result | + +The `references` field may contain multiple `reference` entries, each consisting of the following fields: + +| Field | Description | +| ----------- | ------------------------------------------------------------------ | +| name | the value of the Name field in the draft metadata. | +| id | the CMR identifier for the result | +| location | the URL at which the full metadata for the result can be retrieved | +| revision-id | the internal CMR version number for the result | + +__Example__ + +``` + curl -H "Cmr-Pretty: true" \ + "%CMR-ENDPOINT%/grid-drafts.xml?name=grid-name" +``` + +__Sample response__ + +``` + + + 1 + 13 + + + grid-name + GD1200000000-PROV1 + %CMR-ENDPOINT%/concepts/GD1200000000-PROV1/4 + 4 + + + +``` + +##### JSON + +The JSON response includes the following fields. + +* hits - How many total Order Options were found. +* took - How long the search took in milliseconds +* items - a list of the current page of Order Options with the following fields + * concept\_id + * revision\_id + * provider\_id + * native\_id + * name + +__Example__ + +``` + curl -H "Cmr-Pretty: true" \ + "%CMR-ENDPOINT%/grid-drafts.json?name=grid-name" +``` + +__Sample response__ + +``` + { + "hits": 1, + "took": 10, + "items": [ + { + "concept_id": "GD1200000000-PROV1", + "revision\_id": 4, + "provider\_id": "PROV-1", + "native\_id": "sampleNative-Id", + "name": "grid-name" + } + ] + } +``` + +##### UMM JSON + +The UMM JSON response contains meta-metadata of the Variable Draft, the UMM fields and the associations field if applicable. [To search over specific versions of UMM](#umm-json). + +__Example__ + +``` + curl -H "pretty=true" \ + "%CMR-ENDPOINT%/grid-drafts.umm_json?name=grid-name" +``` + +__Sample response__ + +``` + { + "hits": 1, + "took": 17, + "items": [ + { + "meta": { + "revision-id": 1, + "deleted": false, + "provider-id": "PROV1", + "user-id": "exampleuser", + "native-id": "samplenativeid12", + "concept-id": "GD1200000000-PROV1", + "revision-date": "2022-10-26T19:17:27.021Z", + "concept-type": "grid-draft" + }, + "umm": { + "MetadataSpecification": { + "URL": "https://cdn.earthdata.nasa.gov/generic/grid/v0.0.1", + "Name": "Grid", + "Version": "0.0.1" + }, + "Name": "Grid-A7-v1", + "LongName": "Grid A-7 version 1.0", + "Version": "v1.0", + "Description": "A sample grid", + "GridDefinition": { + "CoordinateReferenceSystemID": { + "Type": "EPSG", + "Code": "EPSG:4326", + "Title": "WGS84 - World Geodetic System 1984, used in GPS - EPSG:4326", + "URL": "https://epsg.io/4326" + }, + "DimensionSize": { + "Height": 3.14, + "Width": 3.14, + "Time": "12:00:00Z", + "Other": { + "Name": "Other Dimension Size", + "Value": "42", + "Description": "Details about the other dimension size." + } + }, + "Resolution": { + "Unit": "Meter", + "LongitudeResolution": 64, + "LatitudeResolution": 32 + }, + "SpatialExtent": { + "0_360_DegreeProjection": false, + "NorthBoundingCoordinate": -90.0, + "EastBoundingCoordinate": 180.0, + "SouthBoundingCoordinate": 90.0, + "WestBoundingCoordinate": -180.0 + }, + "ScaleExtent": { + "ScaleDimensions": [{ + "Unit": "Meter", + "0_360_DegreeProjection": true, + "Y-Dimension": 0, + "X-Dimension": 30 + }] + } + } + } + } + ] + } +``` + +#### Sorting Grid Draft Results + +By default, Grid Draft results are sorted by name, then by provider-id. + +One or more sort keys can be specified using the sort_key[] parameter. The order used impacts searching. Fields can be prepended with a - to sort in descending order. Ascending order is the default but + (Note: + must be URL encoded as %2B) can be used to explicitly request ascending. + +##### Valid Grid Sort Keys + +* name +* provider +* revision_date + +Examples of sorting by name in descending (reverse alphabetical) and ascending orders (Note: the `+` must be escaped with %2B): + +``` + curl "%CMR-ENDPOINT%/grid-drafts?sort_key\[\]=-name" + curl "%CMR-ENDPOINT%/grid-drafts?sort_key\[\]=%2Bname" +``` \ No newline at end of file From 207d1f95805ee8b416bf63a4da6d11176395f99f Mon Sep 17 00:00:00 2001 From: Erich Reiter Date: Thu, 24 Aug 2023 06:31:10 -0400 Subject: [PATCH 2/3] CMR-9392 Fixing missed typos from PR. --- schemas/resources/schemas/grid-draft/v1.0.0/metadata.json | 2 +- schemas/resources/schemas/grid-draft/v1.0.0/search.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/schemas/resources/schemas/grid-draft/v1.0.0/metadata.json b/schemas/resources/schemas/grid-draft/v1.0.0/metadata.json index 59c6c15396..e114c8ace8 100644 --- a/schemas/resources/schemas/grid-draft/v1.0.0/metadata.json +++ b/schemas/resources/schemas/grid-draft/v1.0.0/metadata.json @@ -115,4 +115,4 @@ "Description": "Sample", "DataType": "STRING" } -} \ No newline at end of file +} diff --git a/schemas/resources/schemas/grid-draft/v1.0.0/search.md b/schemas/resources/schemas/grid-draft/v1.0.0/search.md index 66ce43eb82..1650f6277f 100644 --- a/schemas/resources/schemas/grid-draft/v1.0.0/search.md +++ b/schemas/resources/schemas/grid-draft/v1.0.0/search.md @@ -81,9 +81,9 @@ __Sample response__ The JSON response includes the following fields. -* hits - How many total Order Options were found. +* hits - How many total records were found. * took - How long the search took in milliseconds -* items - a list of the current page of Order Options with the following fields +* items - a list of the current page of records with the following fields * concept\_id * revision\_id * provider\_id @@ -117,7 +117,7 @@ __Sample response__ ##### UMM JSON -The UMM JSON response contains meta-metadata of the Variable Draft, the UMM fields and the associations field if applicable. [To search over specific versions of UMM](#umm-json). +The UMM JSON response contains meta-metadata of the Grid Draft, the UMM fields and the associations field if applicable. [To search over specific versions of UMM](#umm-json). __Example__ From 1c017e8578d9ef2e19e2a39da1f1bf476a21ed62 Mon Sep 17 00:00:00 2001 From: Erich Reiter Date: Thu, 24 Aug 2023 09:00:17 -0400 Subject: [PATCH 3/3] CMR-9392 adding comment to clarify an if statement. --- ingest-app/src/cmr/ingest/api/generic_documents.clj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ingest-app/src/cmr/ingest/api/generic_documents.clj b/ingest-app/src/cmr/ingest/api/generic_documents.clj index 43b2ea68dd..6ac0e2fd08 100644 --- a/ingest-app/src/cmr/ingest/api/generic_documents.clj +++ b/ingest-app/src/cmr/ingest/api/generic_documents.clj @@ -114,6 +114,8 @@ content-type (get headers "content-type") {:keys [spec-key spec-version document-name format]} (pull-metadata-specific-information request-context concept-type content-type raw-document)] + ;; Check to see if the passed in record contains the MetadataSpecification/Name field and its + ;; value matches that from the concept name in the route parameters. (if (and (not= concept-type spec-key) (not= (concept-type draft-concept->spec-map) spec-key) (not= (common-concepts/get-concept-type-of-draft concept-type) spec-key))