Merge pull request #52 from usnistgov/integration2

oar-metadata version 2: python 3 support for re-engineered MIDAS

docker/dockbuild.sh

@@ -10,10 +10,10 @@ execdir=`dirname $0`
 codedir=`(cd $execdir/.. > /dev/null 2>&1; pwd)`
 set -e
 
-## These are set by default via _run.sh; if necessary, uncomment and customize
+## These are set by default via _dockbuild.sh; if necessary, uncomment and customize
 #
 PACKAGE_NAME=oar-metadata
-# 
+
 ## list the names of the image directories (each containing a Dockerfile) for
 ## containers to be built.  List them in dependency order (where a latter one
 ## depends the former ones).  

docker/ejsonschema/Dockerfile

@@ -1,18 +1,32 @@
 FROM oar-metadata/jqfromsrc:latest
 
-RUN apt-get update && apt-get install -y python python-pip python-dev unzip \
-                                         uwsgi uwsgi-plugin-python python-yaml
-RUN pip install 'pip==20.3.4' 'setuptools==44.0.0'
-RUN pip install json-spec jsonmerge==1.3.0 jsonschema==2.6.0 requests pynoid \
-                pytest==4.6.5 filelock crossrefapi
+RUN apt-get update && apt-get install -y unzip uwsgi uwsgi-src \
+                                         uuid-dev libcap-dev libpcre3-dev python3-distutils
+RUN PYTHON=python3.8 uwsgi --build-plugin "/usr/src/uwsgi/plugins/python python38" && \
+    mv python38_plugin.so /usr/lib/uwsgi/plugins/python38_plugin.so && \
+    chmod 644 /usr/lib/uwsgi/plugins/python38_plugin.so
+
+RUN update-alternatives --install /usr/lib/uwsgi/plugins/python3_plugin.so \
+    python_plugin.so /usr/lib/uwsgi/plugins/python38_plugin.so 1
+
+RUN python -m pip install setuptools --upgrade
+RUN python -m pip install json-spec jsonschema==2.4.0 requests \
+                          pytest==4.6.5 filelock crossrefapi pyyaml
+RUN python -m pip install --no-dependencies jsonmerge==1.3.0 
 
 WORKDIR /root
 
 RUN curl -L -o ejsonschema.zip \
     https://github.com/usnistgov/ejsonschema/archive/master.zip && \
     unzip ejsonschema.zip && \
     cd ejsonschema-master && \
-    python setup.py install
+    python setup.py install --install-purelib=/usr/local/lib/python3.8/dist-packages
+
+RUN curl -L -o pynoid.zip \
+    https://github.com/RayPlante/pynoid/archive/master.zip && \
+    unzip pynoid.zip && \
+    cd pynoid-master && \
+    python setup.py install --install-purelib=/usr/local/lib/python3.8/dist-packages
 
 CMD ["bash"]
 

docker/jqfromsrc/Dockerfile

@@ -3,14 +3,16 @@ From oar-metadata/pymongo
 RUN apt-get update && \
     apt-get install -y libonig-dev curl build-essential libtool zip \
                        unzip autoconf git
+RUN pip install pipenv
 
 WORKDIR /root
 RUN git clone http://github.com/stedolan/jq.git jq-dev && \
     cd jq-dev && \
-    git checkout 80052e5275ae8c45b20411eecdd49c945a64a412 && \
+    git checkout a9f97e9e61a910a374a5d768244e8ad63f407d3e && \
     git submodule update --init && \
     autoreconf -fi && \
-    ./configure --with-oniguruma=builtin --disable-docs && \
+    (cd docs && pipenv install) && \
+    ./configure --with-oniguruma=builtin && \
     make -j8 && \
     make check-TESTS && \
     make install

docker/mdtests/Dockerfile

@@ -10,9 +10,12 @@ RUN set -ex; \
    "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$arch"; \
     wget -O /usr/local/bin/gosu.asc \
 "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$arch.asc";\
-    verify-asc.sh /usr/local/bin/gosu /usr/local/bin/gosu.asc \
-                  B42F6819007F00F88E364FD4036A9C25BF357DD4; \
-    rm /usr/local/bin/gosu.asc; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    echo "disable-ipv6" >> "$GNUPGHOME/dirmngr.conf"; \
+    gpg --batch --keyserver hkps://keys.openpgp.org \
+        --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \
+    gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \
+    rm -r /usr/local/bin/gosu.asc; \
     chmod +x /usr/local/bin/gosu; \
     gosu nobody true
 

docker/mdtests/mdtests.sh

@@ -8,7 +8,7 @@ function launch_uwsgi {
     mkdir docker/_docker_ingest_archive
 
     echo starting uwsgi...
-    uwsgi --daemonize docker/_docker_uwsgi.log --plugin python    \
+    uwsgi --daemonize docker/_docker_uwsgi.log --plugin python3   \
           --http-socket :9090 --wsgi-file scripts/ingest-uwsgi.py \
           --set-ph oar_config_file=docker/mdtests/ingest_conf.yml \
           --pidfile /tmp/ingest.pid

docker/pymongo/Dockerfile

@@ -1,11 +1,18 @@
-FROM mongo:4.2
+# This provides the base support for Python 3.8 and MongoDB 4.4 
+
+FROM mongo:4.4
 # VOLUME /data
 MAINTAINER Ray Plante <[email protected]>
 COPY mongod.conf /etc/mongod.conf
 COPY mongod_ctl.sh /usr/local/bin
 
-RUN sed -e '/jessie-updates/ s/^deb/#deb/' /etc/apt/sources.list \
-    > /tmp/sources.list && mv /tmp/sources.list /etc/apt/sources.list
-RUN apt-get update && apt-get install -y python python-pip python-dev
-RUN pip install pymongo
+RUN apt-get update && apt-get install -y ca-certificates locales python3.8 python3-pip python3.8-dev
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1; \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1; \
+    update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1
+RUN locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+RUN python -m pip install pymongo
 

etc/merge/pdp0/Component-schema.json

@@ -0,0 +1,5 @@
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "id": "https://www.nist.gov/od/dm/nerdm-schema/v0.6/Component#",
+    "$ref": "https://www.nist.gov/od/dm/nerdm-schema/v0.6#/definitions/Component"
+}

etc/merge/pdp0/DataFile-schema.json

@@ -0,0 +1,5 @@
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "id": "https://www.nist.gov/od/dm/nerdm-schema/v0.1/DataFile#",
+    "$ref": "https://www.nist.gov/od/dm/nerdm-schema/v0.1#/definitions/Component"
+}

etc/merge/pdp0/README.md

@@ -0,0 +1,12 @@
+This directory contains annotated schemas used to merge NERDm
+documents.  Specifically, it contains schemas for the **pdp0** 
+conventions for merging metadata annotations with the base metadata as
+done in the PDP implementation of SIP publishing.  In the PDP
+implementation, annotations are used to hold metadata values fixed by
+the convention, preventing SIP-providing clients from updating them.  
+
+Because of limitations of jsonmerge (including its lack of awareness
+of ejsonschema conventions), the core NERDm schema is represented by
+`nerdm-amalgamated-schema.json` which folds different resource and 
+and component types together.
+

etc/merge/pdp0/Resource-schema.json

@@ -0,0 +1,5 @@
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "id": "https://www.nist.gov/od/dm/nerdm-schema/v0.6/Resource#",
+    "$ref": "https://www.nist.gov/od/dm/nerdm-schema/v0.6#/definitions/Resource"
+}