Merge updates from 'usnistgov/integration' into rel/1.0.X (doi mintin…

…g, etc.)
usnistgov · Sep 11, 2020 · 9419d2d · 9419d2d
2 parents f99d9e4 + 8657a3b
commit 9419d2d
Show file tree

Hide file tree

Showing 28 changed files with 21,083 additions and 812 deletions.
diff --git a/jq/pod2nerdm.jq b/jq/pod2nerdm.jq
@@ -21,11 +21,11 @@ include "urldecode";
 
 # the base NERDm JSON schema namespace
 #
-def nerdm_schema:  "https://data.nist.gov/od/dm/nerdm-schema/v0.2#";
+def nerdm_schema:  "https://data.nist.gov/od/dm/nerdm-schema/v0.3#";
 
 # the base NERDm JSON schema namespace
 #
-def nerdm_pub_schema:  "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.2#";
+def nerdm_pub_schema:  "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.3#";
 
 # the NERDm context location
 #
@@ -39,6 +39,27 @@ def dciteRefType: nerdm_schema + "/definitions/DCiteReference";
 #
 def resid:  if $id then $id else null end;
 
+# the base URL for the PDR-generated landing page
+#
+def pdrLandingPageBaseURL: "https://data.nist.gov/od/id/";
+
+# extract the local identifier from the EDI-ID.  If the input is an ARK identifier, the scheme and 
+# prefix are dropped
+# 
+# Input: string
+# Output: string
+def ediid2localid:
+    if test("^ark:/\\d+/") then
+        sub("ark:/\\d+/"; "")
+    else . end
+;
+
+# the full URL for PDR-generated landing page, given the EDI-ID
+#
+# Input: string
+# Output: string
+def pdrLandingPageURL:  ediid2localid | (pdrLandingPageBaseURL + .);
+
 # extract the path component from a URI
 #
 # Input: string
@@ -209,7 +230,7 @@ def dist2download:
     .["filepath"] = ( .downloadURL | filepath ) |
     .["@type"] = [ "nrdp:DataFile", "nrdp:DownloadableFile", "dcat:Distribution" ] |
     .["@id"] = (. | componentID("cmps/")) |
-    .["_extensionSchemas"] = [ "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.2#/definitions/DataFile" ] |
+    .["_extensionSchemas"] = [ "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.3#/definitions/DataFile" ] |
     if .format then .format = { description: .format } else . end
 ;
 
@@ -223,7 +244,7 @@ def dist2checksum:
     .["filepath"] = ( .downloadURL | filepath ) |
     .["@type"] = [ "nrdp:ChecksumFile", "nrdp:DownloadableFile", "dcat:Distribution" ] |
     .["@id"] = (. | componentID("cmps/")) |
-    .["_extensionSchemas"] = [ "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.2#/definitions/ChecksumFile" ] |
+    .["_extensionSchemas"] = [ "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.3#/definitions/ChecksumFile" ] |
     .["mediaType"] = "text/plain" |
     .["algorithm"] = { "@type": "Thing", tag: (.filepath|extension) } |
     if .description then . else .["description"] = "SHA-256 checksum value for "+(.filepath|basename|remove_extension) end | 
@@ -263,7 +284,7 @@ def dist2inaccess:
 def dist2accesspage:
     .["@type"] = [ "nrdp:AccessPage", "dcat:Distribution" ] |
     .["@id"] = (. | componentID("#")) |
-    .["_extensionSchemas"] = [ "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.2#/definitions/AccessPage" ] |
+    .["_extensionSchemas"] = [ "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.3#/definitions/AccessPage" ] |
     if .format then .format = { description: .format } else . end
 ;
 
@@ -522,6 +543,7 @@ def podds2resource:
     if .references then .references = (.references | map(cvtref)) else del(.references) end |
     if .components then .components = (.components | map(dist2comp) | insert_subcoll_comps) else del(.components) end |
     if .doi then . else del(.doi) end |
+    if .landingPage then . else .landingPage = (.ediid | pdrLandingPageURL) end | 
     if .theme then .theme = [.theme|.[]|gsub("->"; ":")] else del(.theme) end |
     if .topic then . else del(.topic) end |
     if .issued then . else del(.issued) end |

diff --git a/jq/tests/data/minimal_pod.json b/jq/tests/data/minimal_pod.json
@@ -0,0 +1,25 @@
+{
+    "@type": "dcat:Dataset",
+    "title": "A Minimal Record",
+    "identifier": "EBC9DB05EDF05B0EE043065706812DF87",
+    "contactPoint": {
+        "@type": "vcard:Contact",
+        "fn": "Bob",
+        "hasEmail": "mailto:[email protected]"
+    },
+    "description": "This is a minimal description.",
+    "keyword": [ "testing" ],
+    "landingPage": null,
+    "modified": "2014-03-05",
+    "accessLevel": "public",
+    "bureauCode": [
+        "006:55"
+    ],
+    "programCode": [
+        "006:052"
+    ],
+    "publisher": {
+        "@type": "org:Organization",
+        "name": "National Institute of Standards and Technology"
+    }
+}
diff --git a/jq/tests/test_pod2nerdm.jqt b/jq/tests/test_pod2nerdm.jqt
@@ -10,14 +10,14 @@
 #
 include "pod2nerdm"; nerdm_schema
 null
-"https://data.nist.gov/od/dm/nerdm-schema/v0.2#"
+"https://data.nist.gov/od/dm/nerdm-schema/v0.3#"
 
 #--------------
 # testing nerdm_schema()
 #
 include "pod2nerdm"; nerdm_pub_schema
 null
-"https://data.nist.gov/od/dm/nerdm-schema/pub/v0.2#"
+"https://data.nist.gov/od/dm/nerdm-schema/pub/v0.3#"
 
 #--------------
 # testing nerdm_context()
@@ -31,7 +31,7 @@ null
 #
 include "pod2nerdm"; dciteRefType
 null
-"https://data.nist.gov/od/dm/nerdm-schema/v0.2#/definitions/DCiteReference"
+"https://data.nist.gov/od/dm/nerdm-schema/v0.3#/definitions/DCiteReference"
 
 #--------------
 # testing resid()
@@ -44,12 +44,54 @@ include "pod2nerdm"; def resid: "ID"; resid
 null
 "ID"
 
+#--------------
+# testing pdrLandingPageBaseURL()
+#
+include "pod2nerdm"; pdrLandingPageBaseURL
+null
+"https://data.nist.gov/od/id/"
+
+#--------------
+# testing ediid2localid()
+#
+include "pod2nerdm"; ediid2localid
+"3A1EE2F169DD3B8CE0531A570681DB5D1491"
+"3A1EE2F169DD3B8CE0531A570681DB5D1491"
+
+#--------------
+# testing ediid2localid()
+#
+include "pod2nerdm"; ediid2localid
+"mds2-1491"
+"mds2-1491"
+
+#--------------
+# testing ediid2localid()
+#
+include "pod2nerdm"; ediid2localid
+"ark:/88844/pdr2-1491"
+"pdr2-1491"
+
+#--------------
+# testing pdrLandingPageURL()
+#
+include "pod2nerdm"; pdrLandingPageURL
+"sdi2487gha1/v1"
+"https://data.nist.gov/od/id/sdi2487gha1/v1"
+
+#--------------
+# testing pdrLandingPageURL()
+#
+include "pod2nerdm"; pdrLandingPageURL
+"ark:/88444/pdr0-sdi2487gha1"
+"https://data.nist.gov/od/id/pdr0-sdi2487gha1"
+
 #--------------
 # testing cvtref()
 #
 include "pod2nerdm"; map(cvtref)
 [ "http://goob.net/doc1.txt", "https://goob.gov/doc2.txt" ]
-[{ "@type": ["deo:BibliographicReference"],"@id":"#ref:doc1.txt", "refType": "IsReferencedBy", "_extensionSchemas": [ "https://data.nist.gov/od/dm/nerdm-schema/v0.2#/definitions/DCiteReference" ], "location": "http://goob.net/doc1.txt"}, { "@type": ["deo:BibliographicReference"],"@id":"#ref:doc2.txt", "refType": "IsReferencedBy", "_extensionSchemas": [ "https://data.nist.gov/od/dm/nerdm-schema/v0.2#/definitions/DCiteReference" ], "location": "https://goob.gov/doc2.txt"}]
+[{ "@type": ["deo:BibliographicReference"],"@id":"#ref:doc1.txt", "refType": "IsReferencedBy", "_extensionSchemas": [ "https://data.nist.gov/od/dm/nerdm-schema/v0.3#/definitions/DCiteReference" ], "location": "http://goob.net/doc1.txt"}, { "@type": ["deo:BibliographicReference"],"@id":"#ref:doc2.txt", "refType": "IsReferencedBy", "_extensionSchemas": [ "https://data.nist.gov/od/dm/nerdm-schema/v0.3#/definitions/DCiteReference" ], "location": "https://goob.gov/doc2.txt"}]
 
 #---------------
 # testing filepath()
@@ -178,14 +220,14 @@ include "pod2nerdm"; map(componentID("#"))
 #
 include "pod2nerdm"; dist2download
 {"describedBy": "http://data.nist.gov/srd/srd_data/srd13_janaf-data-defs.json", "downloadURL": "http://data.nist.gov/srd/srd_data/srd13_B-101.json", "mediaType": "application/json","title": "Titanium Boride" }
-{"describedBy": "http://data.nist.gov/srd/srd_data/srd13_janaf-data-defs.json", "downloadURL": "http://data.nist.gov/srd/srd_data/srd13_B-101.json","mediaType": "application/json", "title": "Titanium Boride", "filepath":"srd13_B-101.json", "@type": ["nrdp:DataFile","nrdp:DownloadableFile","dcat:Distribution"],"@id":"cmps/srd13_B-101.json","_extensionSchemas": ["https://data.nist.gov/od/dm/nerdm-schema/pub/v0.2#/definitions/DataFile"]}
+{"describedBy": "http://data.nist.gov/srd/srd_data/srd13_janaf-data-defs.json", "downloadURL": "http://data.nist.gov/srd/srd_data/srd13_B-101.json","mediaType": "application/json", "title": "Titanium Boride", "filepath":"srd13_B-101.json", "@type": ["nrdp:DataFile","nrdp:DownloadableFile","dcat:Distribution"],"@id":"cmps/srd13_B-101.json","_extensionSchemas": ["https://data.nist.gov/od/dm/nerdm-schema/pub/v0.3#/definitions/DataFile"]}
 
 #--------------
 # testing dist2checksum()
 #
 include "pod2nerdm"; dist2checksum
 {"downloadURL": "http://data.nist.gov/srd/srd_data/srd13_B-101.json.sha256", "title": "Checksum for srd13_B-101.json" }
-{"downloadURL": "http://data.nist.gov/srd/srd_data/srd13_B-101.json.sha256","mediaType": "text/plain", "description": "SHA-256 checksum value for srd13_B-101.json", "title": "Checksum for srd13_B-101.json", "filepath":"srd13_B-101.json.sha256", "algorithm": {"@type": "Thing","tag": "sha256"},"@type": ["nrdp:ChecksumFile","nrdp:DownloadableFile","dcat:Distribution"],"@id":"cmps/srd13_B-101.json.sha256","_extensionSchemas": ["https://data.nist.gov/od/dm/nerdm-schema/pub/v0.2#/definitions/ChecksumFile"]}
+{"downloadURL": "http://data.nist.gov/srd/srd_data/srd13_B-101.json.sha256","mediaType": "text/plain", "description": "SHA-256 checksum value for srd13_B-101.json", "title": "Checksum for srd13_B-101.json", "filepath":"srd13_B-101.json.sha256", "algorithm": {"@type": "Thing","tag": "sha256"},"@type": ["nrdp:ChecksumFile","nrdp:DownloadableFile","dcat:Distribution"],"@id":"cmps/srd13_B-101.json.sha256","_extensionSchemas": ["https://data.nist.gov/od/dm/nerdm-schema/pub/v0.3#/definitions/ChecksumFile"]}
 
 #--------------
 # testing dist2hidden()
@@ -206,7 +248,7 @@ include "pod2nerdm"; dist2inaccess
 #
 include "pod2nerdm"; dist2accesspage
 {"accessURL": "https://doi.org/10.18434/T42C7D","title": "A Library to Enable the Modeling of Optical Imaging of Finite Multi-Line Arrays"}
-{"accessURL": "https://doi.org/10.18434/T42C7D","title": "A Library to Enable the Modeling of Optical Imaging of Finite Multi-Line Arrays","@type": [ "nrdp:AccessPage", "dcat:Distribution" ],"@id":"#10.18434/T42C7D","_extensionSchemas":["https://data.nist.gov/od/dm/nerdm-schema/pub/v0.2#/definitions/AccessPage"]}
+{"accessURL": "https://doi.org/10.18434/T42C7D","title": "A Library to Enable the Modeling of Optical Imaging of Finite Multi-Line Arrays","@type": [ "nrdp:AccessPage", "dcat:Distribution" ],"@id":"#10.18434/T42C7D","_extensionSchemas":["https://data.nist.gov/od/dm/nerdm-schema/pub/v0.3#/definitions/AccessPage"]}
 
 #--------------
 # testing dist2comp()
@@ -225,7 +267,7 @@ include "pod2nerdm"; dist2comp
 #
 include "pod2nerdm"; dist2comp
 {"describedBy": "http://data.nist.gov/srd/srd_data/srd13_janaf-data-defs.json", "downloadURL": "http://data.nist.gov/srd/srd_data/srd13_B-101.json", "mediaType": "application/json","title": "Titanium Boride" }
-{"describedBy": "http://data.nist.gov/srd/srd_data/srd13_janaf-data-defs.json", "downloadURL": "http://data.nist.gov/srd/srd_data/srd13_B-101.json","mediaType": "application/json", "title": "Titanium Boride", "filepath":"srd13_B-101.json", "@type": ["nrdp:DataFile","nrdp:DownloadableFile","dcat:Distribution"],"@id":"cmps/srd13_B-101.json","_extensionSchemas": ["https://data.nist.gov/od/dm/nerdm-schema/pub/v0.2#/definitions/DataFile"]}
+{"describedBy": "http://data.nist.gov/srd/srd_data/srd13_janaf-data-defs.json", "downloadURL": "http://data.nist.gov/srd/srd_data/srd13_B-101.json","mediaType": "application/json", "title": "Titanium Boride", "filepath":"srd13_B-101.json", "@type": ["nrdp:DataFile","nrdp:DownloadableFile","dcat:Distribution"],"@id":"cmps/srd13_B-101.json","_extensionSchemas": ["https://data.nist.gov/od/dm/nerdm-schema/pub/v0.3#/definitions/DataFile"]}
 
 #--------------
 # testing dist2comp()
@@ -235,7 +277,7 @@ include "pod2nerdm"; dist2comp
 #
 include "pod2nerdm"; dist2comp
 {"accessURL": "http://www.nsrl.nist.gov/Downloads.htm","conformsTo": "http://www.nsrl.nist.gov/Documents/Data-Formats-of-the-NSRL-Reference-Data-Set-16.pdf","downloadURL": "http://www.nsrl.nist.gov/RDS/rds_2.50/RDS_250.iso","format": "ISO 9660 disk image","mediaType": "application/zip" }
-{"accessURL": "http://www.nsrl.nist.gov/Downloads.htm","conformsTo": "http://www.nsrl.nist.gov/Documents/Data-Formats-of-the-NSRL-Reference-Data-Set-16.pdf","downloadURL": "http://www.nsrl.nist.gov/RDS/rds_2.50/RDS_250.iso","format": { "description": "ISO 9660 disk image"},"mediaType": "application/zip", "filepath":"RDS_250.iso", "@type": ["nrdp:DataFile","nrdp:DownloadableFile","dcat:Distribution"],"@id":"cmps/RDS_250.iso","_extensionSchemas": ["https://data.nist.gov/od/dm/nerdm-schema/pub/v0.2#/definitions/DataFile"] }
+{"accessURL": "http://www.nsrl.nist.gov/Downloads.htm","conformsTo": "http://www.nsrl.nist.gov/Documents/Data-Formats-of-the-NSRL-Reference-Data-Set-16.pdf","downloadURL": "http://www.nsrl.nist.gov/RDS/rds_2.50/RDS_250.iso","format": { "description": "ISO 9660 disk image"},"mediaType": "application/zip", "filepath":"RDS_250.iso", "@type": ["nrdp:DataFile","nrdp:DownloadableFile","dcat:Distribution"],"@id":"cmps/RDS_250.iso","_extensionSchemas": ["https://data.nist.gov/od/dm/nerdm-schema/pub/v0.3#/definitions/DataFile"] }
 
 #--------------
 # testing dist2comp()
@@ -245,7 +287,7 @@ include "pod2nerdm"; dist2comp
 #
 include "pod2nerdm"; dist2comp
 {"accessURL": "http://webbook.nist.gov/chemistry/","description": "Landing page for the NIST Chemistry WebBook.","mediaType": "text/html"}
-{ "accessURL": "http://webbook.nist.gov/chemistry/","description": "Landing page for the NIST Chemistry WebBook.","mediaType": "text/html","@type": ["nrdp:AccessPage","dcat:Distribution"],"@id":"#chemistry/","_extensionSchemas":["https://data.nist.gov/od/dm/nerdm-schema/pub/v0.2#/definitions/AccessPage"]}
+{ "accessURL": "http://webbook.nist.gov/chemistry/","description": "Landing page for the NIST Chemistry WebBook.","mediaType": "text/html","@type": ["nrdp:AccessPage","dcat:Distribution"],"@id":"#chemistry/","_extensionSchemas":["https://data.nist.gov/od/dm/nerdm-schema/pub/v0.3#/definitions/AccessPage"]}
 
 # testing dist2comp()
 #
@@ -347,13 +389,13 @@ include "pod2nerdm"; select_comp_type("nrdp:Subcollection"; "foo/bar")
 #
 include "pod2nerdm"; create_subcoll_for
 "a/b/foo"
-{"@id": "cmps/a/b/foo", "@type": ["nrdp:Subcollection"], "filepath": "a/b/foo", "_extensionSchemas": [ "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.2#/definitions/Subcollection" ]}
+{"@id": "cmps/a/b/foo", "@type": ["nrdp:Subcollection"], "filepath": "a/b/foo", "_extensionSchemas": [ "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.3#/definitions/Subcollection" ]}
 
 # testing insert_subcoll_comps
 #
 include "pod2nerdm"; insert_subcoll_comps
 [{ "title": "Titanium Boride", "filepath": "foo/srd13_B-101.json","@type": ["nrdp:DataFile","nrdp:DownloadableFile","dcat:Distribution"]},{ "title": "foo", "filepath": "foo", "@type": [ "nrdp:Subcollection"]},{"title": "Titanium Boride","filepath": "foo/bar/srd13_B-101.json","@type": ["nrdp:DataFile","nrdp:DownloadableFile","dcat:Distribution"]},{ "title": "foo bar goo", "filepath": "foo/bar/goo", "@type": ["nrdp:Subcollection"]}]
-[{"@id": "cmps/foo/bar", "@type": ["nrdp:Subcollection"], "filepath": "foo/bar", "_extensionSchemas": [ "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.2#/definitions/Subcollection" ]},{ "title": "Titanium Boride", "filepath": "foo/srd13_B-101.json","@type": ["nrdp:DataFile","nrdp:DownloadableFile","dcat:Distribution"]},{ "title": "foo", "filepath": "foo", "@type": [ "nrdp:Subcollection"]},{"title": "Titanium Boride","filepath": "foo/bar/srd13_B-101.json","@type": ["nrdp:DataFile","nrdp:DownloadableFile","dcat:Distribution"]},{ "title": "foo bar goo", "filepath": "foo/bar/goo", "@type": ["nrdp:Subcollection"]}]
+[{"@id": "cmps/foo/bar", "@type": ["nrdp:Subcollection"], "filepath": "foo/bar", "_extensionSchemas": [ "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.3#/definitions/Subcollection" ]},{ "title": "Titanium Boride", "filepath": "foo/srd13_B-101.json","@type": ["nrdp:DataFile","nrdp:DownloadableFile","dcat:Distribution"]},{ "title": "foo", "filepath": "foo", "@type": [ "nrdp:Subcollection"]},{"title": "Titanium Boride","filepath": "foo/bar/srd13_B-101.json","@type": ["nrdp:DataFile","nrdp:DownloadableFile","dcat:Distribution"]},{ "title": "foo bar goo", "filepath": "foo/bar/goo", "@type": ["nrdp:Subcollection"]}]
 
 
 

diff --git a/jq/tests/test_podds2resource.py b/jq/tests/test_podds2resource.py
@@ -3,11 +3,12 @@
 import os, unittest, json, subprocess as subproc, types, pdb
 import ejsonschema as ejs
 
-nerdm = "https://data.nist.gov/od/dm/nerdm-schema/v0.2#"
-nerdmpub = "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.2#"
+nerdm = "https://data.nist.gov/od/dm/nerdm-schema/v0.3#"
+nerdmpub = "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.3#"
 datadir = os.path.join(os.path.dirname(__file__), "data")
 janaffile = os.path.join(datadir, "janaf_pod.json")
 corrfile =  os.path.join(datadir, "CORR-DATA.json")
+minfile =  os.path.join(datadir, "minimal_pod.json")
 pdlfile = os.path.join(datadir, "nist-pdl-oct2016.json")
 jqlib = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
 schemadir = os.path.join(os.path.dirname(jqlib), "model")
@@ -28,7 +29,7 @@ def test_context(self):
 
     def test_schema(self):
         self.assertEquals(self.out['_schema'],
-                          "https://data.nist.gov/od/dm/nerdm-schema/v0.2#")
+                          "https://data.nist.gov/od/dm/nerdm-schema/v0.3#")
     def test_extsch(self):
 
         exts = self.out['_extensionSchemas']
@@ -141,7 +142,7 @@ def test_context(self):
 
     def test_schema(self):
         self.assertEquals(self.out['_schema'],
-                          "https://data.nist.gov/od/dm/nerdm-schema/v0.2#")
+                          "https://data.nist.gov/od/dm/nerdm-schema/v0.3#")
     def test_extsch(self):
 
         exts = self.out['_extensionSchemas']
@@ -230,6 +231,32 @@ def test_theme(self):
         self.assertEqual(theme[1], "Materials characterization")
 
 
+class TestMinimal(unittest.TestCase):  # 
+
+    def setUp(self):
+        # pdb.set_trace() # nerdm::podds2resourc
+        self.out = send_file_thru_jq('nerdm::podds2resource', minfile,
+                                     {"id": "ark:ID"})
+
+    def test_id(self): self.assertEquals(self.out['@id'], "ark:ID")
+    def test_al(self): self.assertEquals(self.out['accessLevel'], "public")
+    def test_context(self):
+        self.assertEquals(self.out['@context'],
+                        [ "https://data.nist.gov/od/dm/nerdm-pub-context.jsonld",
+                          {"@base": "ark:ID"} ])
+
+    def test_arestr(self):
+        props = "title modified ediid landingPage".split()
+        for prop in props:
+            self.assertIn(prop, self.out, "Property not found: " + prop)
+            self.assertIsInstance(self.out[prop], types.StringTypes,
+                "Property '{0}' not a string: {1}".format(prop, self.out[prop]))
+
+    def test_default_landingPage(self):
+        self.assertIsNotNone(self.out.get('landingPage'))
+        self.assertEqual(self.out.get('landingPage'),
+                         "https://data.nist.gov/od/id/EBC9DB05EDF05B0EE043065706812DF87")
+
 class TestValidateNerdm(unittest.TestCase):
 
     def setUp(self):
@@ -250,7 +277,6 @@ def test_with_doi(self):
         out = send_jsonstr_thru_jq('nerdm::podds2resource', ds, {"id": "ark:ID"})
 
         self.val.validate(out, False, True)
-
 
 def format_argopts(argdata):
     """