Merge latest updates (nerdm 0.5) from integration into rel/1.0.X

.travis.yml

@@ -6,6 +6,7 @@ services:
   - docker
 
 before_install:
+  - bash scripts/dhsetup.sh
   - cd docker && bash ./dockbuild.sh
 
 script:

docker/jqfromsrc/Dockerfile

@@ -10,9 +10,9 @@ RUN git clone http://github.com/stedolan/jq.git jq-dev && \
     git checkout 80052e5275ae8c45b20411eecdd49c945a64a412 && \
     git submodule update --init && \
     autoreconf -fi && \
-    ./configure --with-oniguruma=builtin && \
+    ./configure --with-oniguruma=builtin --disable-docs && \
     make -j8 && \
-    make check && \
+    make check-TESTS && \
     make install
 
 CMD ["bash"]

docker/mdtests/Dockerfile

@@ -11,8 +11,8 @@ RUN set -ex; \
 "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$arch.asc";\
     export GNUPGHOME="$(mktemp -d)"; \
     echo "disable-ipv6" >> "$GNUPGHOME/dirmngr.conf"; \
-    gpg --keyserver ha.pool.sks-keyservers.net \
-         --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \
+    gpg --batch --keyserver hkps://keys.openpgp.org \
+        --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \
     gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \
     rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc; \
     chmod +x /usr/local/bin/gosu; \

jq/nerdm2datacite.jq

@@ -353,6 +353,20 @@ def make_ispartof_rel:
     end
 ;
 
+# convert a replacedBy object into an IsObsoletedBy relation
+#
+def make_obsoletedby_rel:
+    if (. and .["@id"] and (.["@id"] | contains("doi"))) then
+      {
+        relatedIdentifier: .["@id"],
+        relatedIdentifierType: "DOI",
+        relationType: "isObsoletedBy"
+      }
+    else
+      empty
+    end
+;
+
 # convert a reference object into a related identifier object
 #
 def make_ref_rel:
@@ -429,7 +443,8 @@ def resource2datacite:
       relatedIdentifiers: [
         (
           (.isPartOf | make_ispartof_rel),
-          (.references | if (.) then (.[] | make_ref_rel) else empty end)
+          (.references | if (.) then (.[] | make_ref_rel) else empty end),
+          (.isReplacedBy | make_obsoletedby_rel)
         )
       ],
       language: "en-US"

jq/pod2nerdm.jq

@@ -21,15 +21,15 @@ include "urldecode";
 
 # the base NERDm JSON schema namespace
 #
-def nerdm_schema:  "https://data.nist.gov/od/dm/nerdm-schema/v0.4#";
+def nerdm_schema:  "https://data.nist.gov/od/dm/nerdm-schema/v0.5#";
 
 # the NERDm pub schema extension namespace
 #
-def nerdm_pub_schema:  "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.4#";
+def nerdm_pub_schema:  "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.5#";
 
 # the NERDm bib schema extension namespace
 #
-def nerdm_bib_schema:  "https://data.nist.gov/od/dm/nerdm-schema/bib/v0.4#";
+def nerdm_bib_schema:  "https://data.nist.gov/od/dm/nerdm-schema/bib/v0.5#";
 
 # the NERDm context location
 #
@@ -170,6 +170,10 @@ def ansc_coll_paths:
     reduce .[] as $ary ([]; .+$ary) | unique
 ;
 
+def shortenDOI:
+    if . then sub("https?://.*doi.org/(doi:)?"; "doi:") else . end
+;
+
 # conversion for a POD-to-NERDm reference node
 #
 # Input: a string containing the reference URL
@@ -298,16 +302,18 @@ def dist2accesspage:
 #
 # Input: a Distribution object
 # Output: a Component object with the detected types given in @type
+# doi:  the "doi:"-prefixed form of the DOI assigned to this records, used to identify
+#       the DOI Access distribution
 #
-def dist2comp: 
+def dist2comp(doi):
     if .downloadURL then
         if (.downloadURL | endswith(".sha256")) then
             dist2checksum
         else
             dist2download
         end
     else if .accessURL then
-        if (.accessURL | test("doi.org")) then
+        if ((.accessURL|shortenDOI) == doi) then
           dist2hidden
         else
           dist2accesspage
@@ -319,7 +325,7 @@ def dist2comp:
 ;
 
 # return the DOI stored in the accessURL, if it exists.  null is returned, if
-# none is found.
+# none is found.   DEPRECATED: DOI passed in via doi property
 #
 # Input: a Distribution object
 # Output:  string: A DOI in in the form of "doi:..."
@@ -510,6 +516,15 @@ def resourceTypes:
     [ isSRD, isPDR, isDCatDS ]
 ;
 
+# Convert a pod status to a NERDm status
+#
+# Input: POD status string or null
+# Output: a NERDm status string
+#
+def cvtstatus:
+    if . == "deactivated" then "removed" else "available" end
+;
+
 # Converts an entire POD Dataset node to a NERDm Resource node
 #
 def podds2resource:
@@ -519,11 +534,12 @@ def podds2resource:
         "_extensionSchemas": [ nerdm_pub_schema + "/definitions/PublicDataResource" ],
         "@type": resourceTypes,
         "@id": resid,
-        "doi": (.distribution + []) | doiFromDist,
+        "doi": .doi | shortenDOI,
         title,
         contactPoint,
         issued,
         modified,
+        status: .status | cvtstatus,
 
         ediid: .identifier,
         landingPage,
@@ -544,8 +560,9 @@ def podds2resource:
         bureauCode,
         programCode
     } |
+    .doi as $doi | 
     if .references then .references = (.references | map(cvtref)) else del(.references) end |
-    if .components then .components = (.components | map(dist2comp) | insert_subcoll_comps) else del(.components) end |
+    if .components then .components = (.components | map(dist2comp($doi)) | insert_subcoll_comps) else del(.components) end |
     if .doi then . else del(.doi) end |
     if .landingPage then . else .landingPage = (.ediid | pdrLandingPageURL) end | 
     if .theme then .theme = [.theme|.[]|gsub("->"; ":")] else del(.theme) end |

jq/tests/data/nist-pdl-oct2016.json

@@ -8987,6 +8987,7 @@
                     "title": "DOI Access for Volatile Data Streat"
                 }
             ],
+            "doi": "http://doi.org/10.18434/T4FK54",
             "identifier": "349FC6C113FA5700E0531A5706810A431464",
             "keyword": [
                 "MTConnect",
@@ -9081,6 +9082,7 @@
                     "title": "Collaborative Guarded-Hot-Plate Tests between the Laboratoire national de m\u00e9trologie et d'essais and the National Institute of Standards and Technology"
                 }
             ],
+            "doi": "https://doi.org/10.18434/T4XK5G",
             "identifier": "3541C38AF67059C5E0531A57068159D51467",
             "keyword": [
                 "bilateral",
@@ -9133,6 +9135,7 @@
                     "title": "Project Data: Wireless Systems for Industrial Environments"
                 }
             ],
+            "doi": "http://doi.org/10.18434/T44S3N",
             "identifier": "356944A3D8670CF4E0531A570681CBA51468",
             "keyword": [
                 "wireless",
@@ -9288,6 +9291,7 @@
                     "title": "SHA-256 file for ASCII version of the code"
                 }
             ],
+            "doi": "http://doi.org/10.18434/T4SW26",
             "identifier": "3A1EE2F169DD3B8CE0531A570681DB5D1491",
             "keyword": [
                 "optical sorting",
@@ -9343,6 +9347,7 @@
                     "title": "Digital Manufacturing Certificate Toolkit"
                 }
             ],
+            "doi": "http://doi.org/10.18434/T4P30K",
             "identifier": "3A28A855F9ABBC7FE0531A570681D98B1492",
             "keyword": [
                 "Trustworthiness",
@@ -9395,6 +9400,7 @@
                     "title": "A Library to Enable the Modeling of Optical Imaging of Finite Multi-Line Arrays"
                 }
             ],
+            "doi": "https://doi.org/10.18434/T42C7D",
             "identifier": "3DA7897F530C7988E0531A570681AAF01502",
             "keyword": [
                 "optical imaging",
@@ -9419,4 +9425,4 @@
         }
     ],
     "describedBy": "https://project-open-data.cio.gov/v1.1/schema/catalog.json"
-}
+}

jq/tests/test_nerdm2datacite.jqt

@@ -352,6 +352,13 @@ include "nerdm2datacite"; make_ref_rel
 { "location": "https://doi.org/10.18434/spd0fjpek351", "title": "Hello!", "refType": "isSupplementTo" }
 { "relatedIdentifier": "https://doi.org/10.18434/spd0fjpek351", "relatedIdentifierType": "DOI", "relationType": "isSupplementTo" }
 
+#--------------
+# testing make_obsoletedby_rel
+#
+include "nerdm2datacite"; make_obsoletedby_rel
+{ "@id": "doi:10.0000/goober", "title": "Hello!" }
+{ "relatedIdentifier": "doi:10.0000/goober", "relatedIdentifierType": "DOI", "relationType": "isObsoletedBy" }
+
 #--------------
 # testing resource2datacite
 #