From bd63cc78e5ebcca1f172df59d46ae28ca6d927a2 Mon Sep 17 00:00:00 2001 From: Ray Plante Date: Wed, 17 Nov 2021 12:24:39 -0500 Subject: [PATCH 1/6] docker/mdtests: refine gosu download verification --- docker/mdtests/Dockerfile | 10 ++++------ docker/mdtests/verify-asc.sh | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 6 deletions(-) create mode 100755 docker/mdtests/verify-asc.sh diff --git a/docker/mdtests/Dockerfile b/docker/mdtests/Dockerfile index 4de5b23..f1ac1b5 100644 --- a/docker/mdtests/Dockerfile +++ b/docker/mdtests/Dockerfile @@ -1,6 +1,7 @@ FROM oar-metadata/ejsonschema RUN apt-get update && apt-get install -y zip wget git +COPY verify-asc.sh /usr/local/bin ENV GOSU_VERSION 1.10 RUN set -ex; \ @@ -9,12 +10,9 @@ RUN set -ex; \ "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$arch"; \ wget -O /usr/local/bin/gosu.asc \ "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$arch.asc";\ - export GNUPGHOME="$(mktemp -d)"; \ - echo "disable-ipv6" >> "$GNUPGHOME/dirmngr.conf"; \ - gpg --batch --keyserver hkps://keys.openpgp.org \ - --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ - gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ - rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + verify-asc.sh /usr/local/bin/gosu /usr/local/bin/gosu.asc \ + B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + rm /usr/local/bin/gosu.asc; \ chmod +x /usr/local/bin/gosu; \ gosu nobody true diff --git a/docker/mdtests/verify-asc.sh b/docker/mdtests/verify-asc.sh new file mode 100755 index 0000000..d120619 --- /dev/null +++ b/docker/mdtests/verify-asc.sh @@ -0,0 +1,28 @@ +#! /bin/bash +# +# verify-asc.sh target target.asc keyid +# +keyservers="hkps://keys.openpgp.org hkps://keyserver.ubuntu.com hkp://keyserver.ubuntu.com hkps://keys.gnupg.net" +# set -x + +export GNUPGHOME="$(mktemp -d)" +echo "disable-ipv6" >> "$GNUPGHOME/dirmngr.conf" + +for keysrvr in $keyservers; do + echo '++' gpg --batch --keyserver $keysrvr --recv-keys $3 + gpg --batch --keyserver $keysrvr --recv-keys $3 && break +done +[ $? == 0 ] || { + echo Failed to retrieve key for id=$3 + [ -e "$GNUPGHOME" ] && rm -r "$GNUPGHOME" + exit 1 +} + +echo '++' gpg --batch --verify $2 $1 +gpg --batch --verify $2 $1 || { + echo "$2": does not verify against $1 + [ -e "$GNUPGHOME" ] && rm -r "$GNUPGHOME" + exit 2 +} + +[ -e "$GNUPGHOME" ] && rm -r "$GNUPGHOME" From 0d50a7ad0a3f44a1fdd22f6085847e4b86626da8 Mon Sep 17 00:00:00 2001 From: Ray Plante Date: Mon, 13 Dec 2021 16:13:16 -0500 Subject: [PATCH 2/6] rmm.ingest.wsgi: add support for a post-commit script --- python/nistoar/rmm/ingest/tests/test_wsgi.py | 58 ++++++++++++- python/nistoar/rmm/ingest/wsgi.py | 91 +++++++++++++++----- 2 files changed, 123 insertions(+), 26 deletions(-) diff --git a/python/nistoar/rmm/ingest/tests/test_wsgi.py b/python/nistoar/rmm/ingest/tests/test_wsgi.py index a6e7650..7461b0b 100644 --- a/python/nistoar/rmm/ingest/tests/test_wsgi.py +++ b/python/nistoar/rmm/ingest/tests/test_wsgi.py @@ -7,11 +7,13 @@ from nistoar.tests import * from nistoar.rmm.ingest import wsgi -pydir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) +testdir = os.path.dirname(os.path.abspath(__file__)) +pydir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(testdir)))) basedir = os.path.dirname(pydir) schemadir = os.path.join(basedir, "model") exdir = os.path.join(schemadir, "examples") janaffile = os.path.join(exdir, "janaf.json") +postcomm = os.path.join(testdir, "postcomm.sh") dburl = None if os.environ.get('MONGO_TESTDB_URL'): @@ -53,15 +55,17 @@ def start(self, status, headers=None, extup=None): def setUp(self): self.archdir = tmpfiles.mkdir("ingest_archive") + self.commitfile = os.path.join(self.archdir, "postcommit.txt") self.config = { "db_url": dburl, 'nerdm_schema_dir': os.path.abspath(schemadir), - 'archive_dir': self.archdir + 'archive_dir': self.archdir, + 'post_commit_exec': postcomm + ' ' + self.commitfile + " {db_url} {recid} {recfile}" } try: self.svc = wsgi.app(self.config) - except Exception, e: + except Exception as e: self.tearDown() raise self.resp = [] @@ -92,6 +96,7 @@ def test_get_types(self): self.assertGreater(len(self.resp), 0) self.assertIn("200", self.resp[0]) self.assertEqual(body[0].strip(), '["nerdm"]') + self.assertFalse(os.path.exists(self.commitfile), "Commit file created unexpectedly") def test_is_ready(self): req = { @@ -103,6 +108,7 @@ def test_is_ready(self): self.assertGreater(len(self.resp), 0) self.assertIn("200", self.resp[0]) self.assertEqual(body[0], 'Service ready\n') + self.assertFalse(os.path.exists(self.commitfile), "Commit file created unexpectedly") def test_auth(self): # test rejection when auth key provided but wsgi is not configured to @@ -201,6 +207,7 @@ def test_is_not_found(self): body = self.svc(req, self.start) self.assertGreater(len(self.resp), 0) self.assertIn("404", self.resp[0]) + self.assertFalse(os.path.exists(self.commitfile), "Commit file created unexpectedly") def test_bad_post_resource(self): with open(janaffile) as doc: @@ -213,6 +220,7 @@ def test_bad_post_resource(self): body = self.svc(req, self.start) self.assertIn("404", self.resp[0]) + self.assertFalse(os.path.exists(self.commitfile), "Commit file created unexpectedly") def test_no_content_length(self): with open(janaffile) as doc: @@ -226,6 +234,7 @@ def test_no_content_length(self): self.assertIn("411", self.resp[0]) self.assertIn("Content-Length", self.resp[0]) + self.assertFalse(os.path.exists(self.commitfile), "Commit file created unexpectedly") def test_bad_content_length(self): with open(janaffile) as doc: @@ -239,7 +248,7 @@ def test_bad_content_length(self): body = self.svc(req, self.start) self.assertIn("400", self.resp[0]) - + self.assertFalse(os.path.exists(self.commitfile), "Commit file created unexpectedly") def test_bad_post_input(self): doc = StringIO('title hello world') @@ -252,6 +261,7 @@ def test_bad_post_input(self): body = self.svc(req, self.start) self.assertIn("400", self.resp[0]) + self.assertFalse(os.path.exists(self.commitfile), "Commit file created unexpectedly") def test_good_post(self): client = MongoClient(dburl) @@ -266,6 +276,7 @@ def test_good_post(self): finally: client.close() + # self.svc = wsgi.app(self.config) with open(janaffile) as doc: clen = len(doc.read()) with open(janaffile) as doc: @@ -282,6 +293,11 @@ def test_good_post(self): self.assertTrue(os.path.isfile(archfile)) self.assertIn("200", self.resp[0]) + self.assertTrue(os.path.isfile(self.commitfile), "Failed to create commit file") + with open(self.commitfile) as fd: + content = fd.read() + self.assertIn("sdp0fjspek351", content) + self.assertIn("mongodb:", content) client = MongoClient(dburl) try: @@ -305,6 +321,40 @@ def setUp(self): def tearDown(self): tmpfiles.clean() + + def test_mkpostcomm(self): + commexec = "echo {recfile} goober {recid} {file}" + commexec = wsgi._mkpostcomm(commexec, file="/tmp/gurn.txt") + self.assertTrue(isinstance(commexec, list), "Output is not a list") + self.assertEqual(len(commexec), 5) + self.assertEqual(commexec[4], "/tmp/gurn.txt") + self.assertEqual(commexec[0], "echo") + self.assertEqual(commexec[1], "{recfile}") + self.assertEqual(commexec[2], "goober") + self.assertEqual(commexec[3], "{recid}") + + commexec = wsgi._mkpostcomm(commexec, "mds2-5555", file="/tmp/gary.txt") + self.assertEqual(commexec[4], "/tmp/gurn.txt") + self.assertEqual(commexec[0], "echo") + self.assertEqual(commexec[1], "{recfile}") + self.assertEqual(commexec[2], "goober") + self.assertEqual(commexec[3], "mds2-5555") + + commexec = wsgi._mkpostcomm(commexec, "mds2-5556", "/tmp", file="/tmp/gary.txt") + self.assertEqual(commexec[4], "/tmp/gurn.txt") + self.assertEqual(commexec[0], "echo") + self.assertEqual(commexec[1], "/tmp/mds2-5556.json") + self.assertEqual(commexec[2], "goober") + self.assertEqual(commexec[3], "mds2-5555") + + commexec = "echo {recfile} goober {recid} {file}" + commexec = wsgi._mkpostcomm(commexec, "mds2-5555", "/tmp", file="/tmp/gary.txt") + self.assertEqual(commexec[4], "/tmp/gary.txt") + self.assertEqual(commexec[0], "echo") + self.assertEqual(commexec[1], "/tmp/mds2-5555.json") + self.assertEqual(commexec[2], "goober") + self.assertEqual(commexec[3], "mds2-5555") + def test_nerdm_archive_cache(self): with open(janaffile) as fd: diff --git a/python/nistoar/rmm/ingest/wsgi.py b/python/nistoar/rmm/ingest/wsgi.py index fb48180..99638e7 100644 --- a/python/nistoar/rmm/ingest/wsgi.py +++ b/python/nistoar/rmm/ingest/wsgi.py @@ -6,7 +6,7 @@ a framework-based implementation if any further capabilities are needed. """ -import os, sys, logging, json, cgi, re +import os, sys, logging, json, cgi, re, subprocess from urlparse import urlsplit, urlunsplit from wsgiref.headers import Headers @@ -83,12 +83,19 @@ def __init__(self, config): else: log.info("Authorization key is required of clients via query parameter") else: - log.warn("No authorization key required of clients") - + log.warning("No authorization key required of clients") + + # check for post-commit script request + self._postexec = config.get('post_commit_exec') + if self._postexec: + try: + self._postexec = _mkpostcomm(self._postexec, '{recid}', **config) + except ValueError as ex: + raise ConfigurationExcetpion("post_commit_exec contains bad formatting") def handle_request(self, env, start_resp): handler = Handler(self._loaders, env, start_resp, - self.archdir, self._auth) + self.archdir, self._auth, self._postexec) return handler.handle() def __call__(self, env, start_resp): @@ -98,7 +105,7 @@ def __call__(self, env, start_resp): class Handler(object): - def __init__(self, loaders, wsgienv, start_resp, archdir, auth=None): + def __init__(self, loaders, wsgienv, start_resp, archdir, auth=None, postexec=None): self._env = wsgienv self._start = start_resp self._meth = wsgienv.get('REQUEST_METHOD', 'GET') @@ -107,6 +114,7 @@ def __init__(self, loaders, wsgienv, start_resp, archdir, auth=None): self._msg = "unknown status" self._auth = auth self._archdir = archdir + self._postexec = postexec self._loaders = loaders @@ -152,7 +160,7 @@ def authorize_via_queryparam(self): # match the last value provided return len(auths) > 0 and self._auth[1] == auths[-1] if len(auths) > 0: - log.warn("Authorization key provided, but none has been configured") + log.warning("Authorization key provided, but none has been configured") return len(auths) == 0 def authorize_via_headertoken(self): @@ -163,7 +171,7 @@ def authorize_via_headertoken(self): return len(parts) > 1 and parts[0] == "Bearer" and \ self._auth[1] == parts[1] if authhdr: - log.warn("Authorization key provided, but none has been configured") + log.warning("Authorization key provided, but none has been configured") return authhdr == "" def send_unauthorized(self): @@ -177,13 +185,14 @@ def do_GET(self, path): path = path.strip('/') if not path: try: - out = json.dumps(self._loaders.keys()) + '\n' - except Exception, ex: + out = json.dumps(list(self._loaders.keys())) + '\n' + except Exception as ex: log.exception("Internal error: "+str(ex)) return self.send_error(500, "Internal error") self.set_response(200, "Supported Record Types") self.add_header('Content-Type', 'application/json') + self.add_header('Content-Length', str(len(out))) self.end_headers() return [out] elif path in self._loaders: @@ -201,7 +210,7 @@ def do_POST(self, path): return self.send_error(405, "POST not supported on this resource") elif len(steps) == 1: if steps[0] == 'nerdm': - return self.post_nerdm_record() + return self.ingest_nerdm_record() else: return self.send_error(403, "new records are not allowed for " + "submission to this resource") @@ -253,7 +262,7 @@ def nerdm_archive_commit(self, arkid): .format(arkid, str(ex))) - def post_nerdm_record(self): + def ingest_nerdm_record(self): """ Accept a NERDm record for ingest into the RMM """ @@ -261,10 +270,10 @@ def post_nerdm_record(self): try: clen = int(self._env['CONTENT_LENGTH']) - except KeyError, ex: + except KeyError as ex: log.exception("Content-Length not provided for input record") return self.send_error(411, "Content-Length is required") - except ValueError, ex: + except ValueError as ex: log.exception("Failed to parse input JSON record: "+str(e)) return self.send_error(400, "Content-Length is not an integer") @@ -272,10 +281,10 @@ def post_nerdm_record(self): bodyin = self._env['wsgi.input'] doc = bodyin.read(clen) rec = json.loads(doc) - except Exception, ex: + except Exception as ex: log.exception("Failed to parse input JSON record: "+str(ex)) - log.warn("Input document starts...\n{0}...\n...{1} ({2}/{3} chars)" - .format(doc[:75], doc[-20:], len(doc), clen)) + log.warning("Input document starts...\n{0}...\n...{1} ({2}/{3} chars)" + .format(doc[:75], doc[-20:], len(doc), clen)) return self.send_error(400, "Failed to load input record (bad format?): "+ str(ex)) @@ -295,14 +304,15 @@ def post_nerdm_record(self): self.end_headers() return [ json.dumps([str(e) for e in res.errs]) + '\n' ] - except RecordIngestError, ex: + except RecordIngestError as ex: log.exception("Failed to load posted record: "+str(ex)) self.set_response(400, "Input record is not valid (missing @id)") self.add_header('Content-Type', 'application/json') self.end_headers() - return [ json.dumps([ "Record is missing @id property" ]) + '\n' ] + out = json.dumps([ "Record is missing @id property" ]) + '\n' + return [ out.encode() ] - except Exception, ex: + except Exception as ex: log.exception("Loading error: "+str(ex)) return self.send_error(500, "Load failure due to internal error") @@ -311,13 +321,50 @@ def post_nerdm_record(self): except Exception as ex: log.exception("Commit error: "+str(ex)) + if self._postexec: + # run post-commit script + try: + self.nerdm_post_commit(recid) + except Exception as ex: + log.exception("Post-commit error: "+str(ex)) + log.info("Accepted record %s with @id=%s", rec.get('ediid','?'), rec.get('@id','?')) self.set_response(200, "Record accepted") self.end_headers() return [] - - + + def nerdm_post_commit(self, recid): + """ + run an external executable for further processing after the record is commited to + the database (e.g. update an external index) + """ + cmd = _mkpostcomm(self._postexec, recid, self._archdir) + + try: + log.debug("Executing post-commit script:\n %s", " ".join(cmd)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (out, err) = p.communicate() + if p.returncode != 0: + log.error("Error occurred while running post-commit script:\n"+(err or out)) + except OSError as ex: + log.error("Failed to execute post-commit script:\n %s\n%s", " ".join(cmd), str(ex)) + except Exception as ex: + log.error("Unexpected failure executing post-commit script:\n %s\n%s", " ".join(cmd), str(ex)) + +def _mkpostcomm(cmd, recid='{recid}', archdir=None, recfile=None, **vals): + if not isinstance(cmd, (list, tuple)): + cmd = cmd.split() + + vals['recid'] = recid + if recfile is None: + recfile = '{recfile}' + if archdir: + recfile = os.path.join(archdir, re.sub(r'^ark:/\d+/', '', recid)+".json") + vals['recfile'] = recfile + + cmd = [arg.format(**vals) for arg in cmd] + return cmd def _get_oar_home(): home = os.environ.get('OAR_HOME') @@ -334,7 +381,7 @@ def _get_oar_home(): if not os.path.exists(os.path.join(home, "etc")): home = os.path.dirname(home) return home - except OSError, ex: + except OSError as ex: log.exception("OSError while looking for OAR_HOME: "+str(e)) return None From 0fc3396467a8ba30613b48f2f8fb24f8f13c4c3a Mon Sep 17 00:00:00 2001 From: Ray Plante Date: Mon, 13 Dec 2021 16:16:43 -0500 Subject: [PATCH 3/6] include (via __init__.py) and fix missing tests --- python/nistoar/doi/resolving/tests/__init__.py | 0 python/nistoar/doi/resolving/tests/test_common.py | 3 +++ python/nistoar/doi/resolving/tests/test_resolving.py | 5 ++++- python/nistoar/doi/tests/__init__.py | 0 python/nistoar/nerdm/convert/tests/test_pod_doi.py | 4 ++++ python/nistoar/rmm/ingest/tests/__init__.py | 0 python/nistoar/rmm/tests/__init__.py | 0 7 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 python/nistoar/doi/resolving/tests/__init__.py create mode 100644 python/nistoar/doi/tests/__init__.py create mode 100644 python/nistoar/rmm/ingest/tests/__init__.py create mode 100644 python/nistoar/rmm/tests/__init__.py diff --git a/python/nistoar/doi/resolving/tests/__init__.py b/python/nistoar/doi/resolving/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/nistoar/doi/resolving/tests/test_common.py b/python/nistoar/doi/resolving/tests/test_common.py index b228ec8..b66c829 100644 --- a/python/nistoar/doi/resolving/tests/test_common.py +++ b/python/nistoar/doi/resolving/tests/test_common.py @@ -10,6 +10,9 @@ "http://github.com/usnistgov/oar-metadata/", "datasupport@nist.gov") +def setUpModule(): + res._client_info = None + class TestFuncs(test.TestCase): def tearDown(self): diff --git a/python/nistoar/doi/resolving/tests/test_resolving.py b/python/nistoar/doi/resolving/tests/test_resolving.py index d8b9da2..dfc2994 100644 --- a/python/nistoar/doi/resolving/tests/test_resolving.py +++ b/python/nistoar/doi/resolving/tests/test_resolving.py @@ -14,7 +14,10 @@ cli = ("NIST Open Access for Research", "testing", "http://github.com/usnistgov/oar-metadata/", "datasupport@nist.gov") -set_client_info(*cli) +def setUpModule(): + set_client_info(*cli) +def tearDownModule(): + set_client_info(None, None, None, None) logger = logging.getLogger("test") diff --git a/python/nistoar/doi/tests/__init__.py b/python/nistoar/doi/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/nistoar/nerdm/convert/tests/test_pod_doi.py b/python/nistoar/nerdm/convert/tests/test_pod_doi.py index dbd3f2e..cfd0007 100644 --- a/python/nistoar/nerdm/convert/tests/test_pod_doi.py +++ b/python/nistoar/nerdm/convert/tests/test_pod_doi.py @@ -38,6 +38,10 @@ "email": "datasupport@nist.gov" } +def setUpModule(): + import nistoar.doi.resolving.common as res + res._client_info = None + class TestConvertAuthors(unittest.TestCase): def test_citeproc_author2nerdm_author(self): diff --git a/python/nistoar/rmm/ingest/tests/__init__.py b/python/nistoar/rmm/ingest/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/nistoar/rmm/tests/__init__.py b/python/nistoar/rmm/tests/__init__.py new file mode 100644 index 0000000..e69de29 From 28e75ddd837e596ff3a33b11b5d9df707ac5ec6c Mon Sep 17 00:00:00 2001 From: Ray Plante Date: Mon, 13 Dec 2021 16:17:48 -0500 Subject: [PATCH 4/6] rmm.ingest.wsgi: add missing post-commit test script --- python/nistoar/rmm/ingest/tests/postcomm.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 python/nistoar/rmm/ingest/tests/postcomm.sh diff --git a/python/nistoar/rmm/ingest/tests/postcomm.sh b/python/nistoar/rmm/ingest/tests/postcomm.sh new file mode 100755 index 0000000..db9500f --- /dev/null +++ b/python/nistoar/rmm/ingest/tests/postcomm.sh @@ -0,0 +1,14 @@ +#! /bin/bash +# +# Used by test_wsgi.py, this script prints arguments to a given output file +# +# USAGE: postcomm.sh OUTFILE [ARG ...] +# +set -e +[ -n "$1" ] || { + echo "${0}: Missing output filename" + exit 1 +} +out=$1; shift + +echo "$@" > $out From 04e17b2cfb73e3c99c0d08e14ca42e4a588dfff0 Mon Sep 17 00:00:00 2001 From: Ray Plante Date: Wed, 15 Dec 2021 15:49:49 -0500 Subject: [PATCH 5/6] enable authentication when updating autocomm table --- python/nistoar/rmm/ingest/wsgi.py | 19 ++++++++++++++++++- scripts/ingest-uwsgi.py | 2 ++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/python/nistoar/rmm/ingest/wsgi.py b/python/nistoar/rmm/ingest/wsgi.py index 99638e7..93da14c 100644 --- a/python/nistoar/rmm/ingest/wsgi.py +++ b/python/nistoar/rmm/ingest/wsgi.py @@ -8,6 +8,7 @@ import os, sys, logging, json, cgi, re, subprocess from urlparse import urlsplit, urlunsplit +from collections import Mapping from wsgiref.headers import Headers from ..mongo.nerdm import (NERDmLoader, LoadLog, @@ -18,6 +19,7 @@ DEF_BASE_PATH = "/" + class RMMRecordIngestApp(object): def __init__(self, config): @@ -352,10 +354,14 @@ def nerdm_post_commit(self, recid): except Exception as ex: log.error("Unexpected failure executing post-commit script:\n %s\n%s", " ".join(cmd), str(ex)) -def _mkpostcomm(cmd, recid='{recid}', archdir=None, recfile=None, **vals): +def _mkpostcomm(cmd, recid='{recid}', archdir=None, recfile=None, **fmtdata): if not isinstance(cmd, (list, tuple)): cmd = cmd.split() + if fmtdata is None: + fmtdata = {} + vals = _data4fmt(fmtdata) + vals['recid'] = recid if recfile is None: recfile = '{recfile}' @@ -366,6 +372,17 @@ def _mkpostcomm(cmd, recid='{recid}', archdir=None, recfile=None, **vals): cmd = [arg.format(**vals) for arg in cmd] return cmd +class _ov(object): + def __init__(self, d): + self.__dict__ = _data4fmt(d) + +def _data4fmt(d): + d = dict(d) + for key in d: + if isinstance(d[key], Mapping): + d[key] = _ov(d[key]) + return d + def _get_oar_home(): home = os.environ.get('OAR_HOME') if home: diff --git a/scripts/ingest-uwsgi.py b/scripts/ingest-uwsgi.py index 478c0e2..3fd6505 100644 --- a/scripts/ingest-uwsgi.py +++ b/scripts/ingest-uwsgi.py @@ -95,6 +95,8 @@ def get_confservice(): acfg['user'] = rmmcfg['oar.mongodb.readwrite.user'] acfg['pass'] = rmmcfg['oar.mongodb.readwrite.password'] + acfg['rouser'] = rmmcfg['oar.mongodb.read.user'] + acfg['ropass'] = rmmcfg['oar.mongodb.read.password'] except Exception, ex: raise ConfigurationException("Failed to retrieve Mongo authentication "+ From 13a67226b1bb550529bc900ef61edcf23dc019f2 Mon Sep 17 00:00:00 2001 From: Ray Plante Date: Fri, 17 Dec 2021 17:00:15 -0500 Subject: [PATCH 6/6] add some documentation to the ingest service wsgi --- python/nistoar/rmm/ingest/wsgi.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/python/nistoar/rmm/ingest/wsgi.py b/python/nistoar/rmm/ingest/wsgi.py index 93da14c..8cd4333 100644 --- a/python/nistoar/rmm/ingest/wsgi.py +++ b/python/nistoar/rmm/ingest/wsgi.py @@ -21,8 +21,36 @@ class RMMRecordIngestApp(object): + """ + This is the WSGI implementation of the NERDm record ingest service. NERDm records submitted to this + service are validated and (if valid) loads them into the RMM. This service can be configured to also + carry out a post-ingest action; this is leveraged by oar-docker to update the SDP's autocomplete index. + + This App accepts the following configuration parameters: + :param str db_url: the (MongoDB) URL of the RMM database to load into + :param dict db_authn: the configuration controlling authentication to the database + :param str db_authn.user: the database user name to connect to the database with + :param str db_authn.pass: the database password (corresponding to `db_authn.user`) to authenticate + to the database with. + :param str db_authn.rm_config_loc: the name of a configuration set to retrieve to load authentication + in from. The values found there (which should include `user` and `pass`) will + be loaded into the `db_authn` configuration. + :param str auth_key: the Bearer token that must be presented as client credentials to the service; + if an incorrect token is included with service requests, the request will + rejected with a 401 status. + :param str archive_dir a directory where accepted records should be stored after ingest. + :param str|list post_commit_exec: a string or list of strings that specify a program and its arguments + that should be run after the record is loaded into the database. If given as + string, it will be split into a list at its spaces to provide the executable and + arguments. The string values an include a words surrounded by braces (e.g. + `{archive_dir}`); those whose word matches a parameter that is part of the + provided configuration will get substituted with the values of the parameters. + """ def __init__(self, config): + """ + instantiate the service with the provided configuration. + """ self.base_path = config.get('base_path', DEF_BASE_PATH) self.dburl = config.get('db_url') if not self.dburl: