diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0ee67495..a0543487 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -35,6 +35,23 @@ jobs:
- name: Install linux deps
run: sudo apt-get install -y libcurl4-openssl-dev
+ - name: Set up Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: 3.8
+
+ - name: Install python dependencies
+ run: pip install jep==3.9 # Install the jep library
+
+ - name: Set JEP_LIBRARY_PATH environment variable
+ run: |
+ echo "JEP_LIBRARY_PATH=$(python -c 'import site; print(site.getsitepackages()[0])')/jep" >> $GITHUB_ENV
+
+ - name: test
+ run: |
+ echo $JEP_LIBRARY_PATH
+ ls $JEP_LIBRARY_PATH
+
- name: Install deps
run: Rscript -e 'install.packages(c("httr", "jsonlite", "remotes"))'
@@ -58,7 +75,7 @@ jobs:
docker-publish:
name: Docker Build and Publish
- if: github.ref_name == 'develop' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref_name, 'bugfix')
+ if: github.ref_name == 'develop' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref_name, 'feature')
needs: maven-build
runs-on: ubuntu-latest
permissions:
diff --git a/.vscode/launch.json b/.vscode/launch.json
index af81a530..3bb1538e 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -20,6 +20,18 @@
".",
"${fileDirname}"
]
+ },
+ {
+ "name": "Run MDQEngine",
+ "type": "java",
+ "request": "launch",
+ "mainClass": "edu.ucsb.nceas.mdqengine.MDQEngine",
+ "args": [
+ "/opt/local/metadig/suites/python-suite.xml",
+ "${workspaceFolder}/src/test/resources/test-docs/doi:10.18739_A2W08WG3R.xml",
+ "${workspaceFolder}/src/test/resources/test-docs/doi:10.18739_A2W08WG3R.sm",
+ ],
+ "cwd": "${workspaceFolder}"
}
]
}
\ No newline at end of file
diff --git a/Docker/metadig-worker/Dockerfile b/Docker/metadig-worker/Dockerfile
index 2fac2518..b1a24c53 100644
--- a/Docker/metadig-worker/Dockerfile
+++ b/Docker/metadig-worker/Dockerfile
@@ -11,7 +11,7 @@ LABEL maintainer="Jeanette Clark jclark@nceas.ucsb.edu"
# Set the working directory
WORKDIR /var/lib/metadig
-ARG ENGINE_TAG=2.0.0
+ARG ENGINE_TAG=3.0.0-SNAPSHOT
ENV ENGINE_TAG=${ENGINE_TAG}
# The most recently built jar file is copied from the maven build directory to this dir by maven, so that
@@ -24,7 +24,11 @@ COPY target/metadig-engine-$ENGINE_TAG.jar metadig-engine-$ENGINE_TAG.jar
COPY Docker/metadig-worker/solr solr/
RUN apt-get update
-RUN apt-get install -y g++ r-base r-base-dev r-base-core r-doc-html libc6-dev libssl-dev libxml2 libxml2-dev libcurl4-openssl-dev
+RUN apt-get install -y g++ git r-base r-base-dev r-base-core r-doc-html libc6-dev libssl-dev libxml2 libxml2-dev libcurl4-openssl-dev python3 python3-pip python3-dev openjdk-17-jdk
+
+# Set Java environment variables
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
+ENV PATH="$PATH:$JAVA_HOME/bin"
# Install required R packages
RUN echo 'options(repos = c(CRAN = "http://cran.rstudio.com"))' >> /usr/lib/R/etc/Rprofile.site
@@ -34,6 +38,9 @@ RUN Rscript -e "remotes::install_github('NCEAS/metadig-r', ref = 'v.0.2.0')"
# DataONE indexer prints copious error msgs if these files don't exist
RUN mkdir -p /etc/dataone/index && touch /etc/dataone/index/d1client.properties && touch /etc/dataone/node.properties && touch /etc/dataone/index/cn-index-processor.properties
+COPY Docker/metadig-worker/requirements.txt /opt/local/metadig/
+RUN pip install -r /opt/local/metadig/requirements.txt
+
# Run the Worker process
# Note: docker --buile-arg only allows one argument (one token only, multiple tokens inside quotes doesn't work, so have
# to specify java options directly on command line.
diff --git a/Docker/metadig-worker/requirements.txt b/Docker/metadig-worker/requirements.txt
new file mode 100644
index 00000000..5186f3f6
--- /dev/null
+++ b/Docker/metadig-worker/requirements.txt
@@ -0,0 +1,2 @@
+numpy
+metadig @ git+https://github.com/NCEAS/metadig-py.git
diff --git a/bin/sendAssessmentTest.py b/bin/sendAssessmentTest.py
index 7f8ecc36..8e0cc543 100755
--- a/bin/sendAssessmentTest.py
+++ b/bin/sendAssessmentTest.py
@@ -1,15 +1,12 @@
-#!/Users/clark/.virtualenvs/metadig/bin/python
-
-# this is a shebang to your python path. needs to be 2.7
-#/usr/bin/python
+#!/usr/bin/python
# This Python program can be used for local testing of RabbitMQ messaging between
# metadig services, assessment processing and Solr indexing.
import sys
import re
-print "Sending assessment request to metadig-controler (test mode):"
-print "Number of arguments: ", len(sys.argv)
+print("Sending assessment request to metadig-controller (test mode):")
+print("Number of arguments: ", len(sys.argv))
import socket
host="localhost"
@@ -24,7 +21,7 @@
clientsocket.connect((host, portNum))
testCount = 1
-suiteId = "test.suite"
+suiteId = "python.suite"
nodeId = "urn:node:ARCTIC"
idVal = "doi:10.18739/A2W08WG3R"
@@ -34,17 +31,17 @@
smFile = "%s/%s.sm" % (testDir, fileIdVal)
# First send the type of request, either 'graph' or 'quality'
-clientsocket.send("%s\n" % 'quality')
+clientsocket.send('quality\n'.encode('utf-8'))
# Next send the number of tests that will be run
-clientsocket.send("%s\n" % testCount)
+clientsocket.send(f"{testCount}\n".encode('utf-8'))
msg = '%s,%s,%s,%s,%s\n' % (idVal, mdFile, smFile, suiteId, nodeId)
print(msg)
-clientsocket.send(msg)
+clientsocket.send(msg.encode('utf-8'))
# Server will stop reading from the current port connection
-clientsocket.send("Done\n")
+clientsocket.send("Done\n".encode('utf-8'))
# Server will stop reading from any port connection, but will continue to run
clientsocket.close()
diff --git a/bin/startController.sh b/bin/startController.sh
index d7064d32..23eb518e 100755
--- a/bin/startController.sh
+++ b/bin/startController.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-version=2.5.0
+version=3.0.0-SNAPSHOT
# Use first argument if sending jobs via port, for testing, for example:
java -cp /opt/local/metadig/config:./target/metadig-engine-${version}.jar edu.ucsb.nceas.mdqengine.Controller 33000
diff --git a/bin/startWorker.sh b/bin/startWorker.sh
index a5c7da61..640ff99c 100755
--- a/bin/startWorker.sh
+++ b/bin/startWorker.sh
@@ -1,10 +1,7 @@
#!/bin/bash
# Start a local copy of metadig-worker for RabbitMQ based development testing.
-version=2.5.0
+version=3.0.0-SNAPSHOT
-# Include the MetaDIG python library
-export JYTHONPATH=/opt/local/metadig/metadig-py
-
-java -cp /opt/local/metadig/config:./target/metadig-engine-${version}.jar:./target/classes/solr -Dpython.path=JYTHONPATH edu.ucsb.nceas.mdqengine.Worker
+java -cp /opt/local/metadig/config:./target/metadig-engine-${version}.jar:./target/classes/solr edu.ucsb.nceas.mdqengine.Worker
diff --git a/helm/metadig-controller/Chart.yaml b/helm/metadig-controller/Chart.yaml
index 564e1d8d..a4eebb89 100644
--- a/helm/metadig-controller/Chart.yaml
+++ b/helm/metadig-controller/Chart.yaml
@@ -15,10 +15,10 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.0.2
+version: 1.0.4
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
-appVersion: "2.5.0"
+appVersion: "3.0.0"
diff --git a/helm/metadig-controller/config.dev/metadig.properties b/helm/metadig-controller/config.dev/metadig.properties
index ada5d878..added0b1 100644
--- a/helm/metadig-controller/config.dev/metadig.properties
+++ b/helm/metadig-controller/config.dev/metadig.properties
@@ -33,4 +33,6 @@ quartz.monitor = true
quartz.monitor.schedule = 0 0 0 * * ?
quartz.monitor.processing.time = 12
quartz.monitor.run.limit = 10
+# jep library
+jep.path = /usr/local/lib/python3.10/dist-packages/jep
diff --git a/helm/metadig-controller/config/metadig.properties b/helm/metadig-controller/config/metadig.properties
index 3674dda1..23044e29 100644
--- a/helm/metadig-controller/config/metadig.properties
+++ b/helm/metadig-controller/config/metadig.properties
@@ -59,3 +59,5 @@ quartz.monitor = true
quartz.monitor.schedule = 0 0 0 * * ?
quartz.monitor.processing.time = 12
quartz.monitor.run.limit = 10
+# jep library
+jep.path = /usr/local/lib/python3.10/dist-packages/jep
diff --git a/helm/metadig-controller/values.yaml b/helm/metadig-controller/values.yaml
index 80931bc7..0643279d 100644
--- a/helm/metadig-controller/values.yaml
+++ b/helm/metadig-controller/values.yaml
@@ -7,7 +7,7 @@ replicaCount: 1
image:
repository: ghcr.io/nceas/metadig-controller
pullPolicy: Always
- tag: "v.2.5.0"
+ tag: "v.3.0.0"
imagePullSecrets: []
diff --git a/helm/metadig-scheduler/Chart.yaml b/helm/metadig-scheduler/Chart.yaml
index 9ae738fa..745af7e7 100644
--- a/helm/metadig-scheduler/Chart.yaml
+++ b/helm/metadig-scheduler/Chart.yaml
@@ -15,10 +15,10 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.0.3
+version: 1.0.4
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
-appVersion: "2.5.0"
+appVersion: "3.0.0"
diff --git a/helm/metadig-scheduler/config.dev/taskList.csv b/helm/metadig-scheduler/config.dev/taskList.csv
index 84b89c2e..86e64ed7 100644
--- a/helm/metadig-scheduler/config.dev/taskList.csv
+++ b/helm/metadig-scheduler/config.dev/taskList.csv
@@ -19,8 +19,8 @@ task-type,task-name,task-group,cron-schedule,params
# - requestType: for score tasks, determine type of portal processing ("portal" or "node")
#
# Dataset quality scoring tasks
-quality,quality-test-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;arctic.data.center.suite.1;urn:node:mnTestARCTIC;2022-05-01T00:00:00.00Z;1;1000"
-quality,quality-test-dataone-fair,metadig,10 0/1 * * * ?,"^eml.*|^http.*eml.*|.*www.isotc211.org.*;FAIR-suite-0.3.1;urn:node:cnStage;2022-05-01T00:00:00.00Z;1;1000"
+quality,quality-test-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;arctic.data.center.suite-1.2.0;urn:node:mnTestARCTIC;2022-05-01T00:00:00.00Z;1;1000"
+quality,quality-test-dataone-fair,metadig,10 0/1 * * * ?,"^eml.*|^http.*eml.*|.*www.isotc211.org.*;FAIR-suite-0.4.0;urn:node:cnStage;2022-05-01T00:00:00.00Z;1;1000"
#
# Portal scoring tasks
score,portal-test-arctic-FAIR,metadig,10 0/1 * * * ?,"*portals*;FAIR-suite-0.3.1;urn:node:mnTestARCTIC;2022-05-01T00:00:00.00Z;1;100;portal"
diff --git a/helm/metadig-scheduler/config/taskList.csv b/helm/metadig-scheduler/config/taskList.csv
index 70a5acbf..989c936b 100644
--- a/helm/metadig-scheduler/config/taskList.csv
+++ b/helm/metadig-scheduler/config/taskList.csv
@@ -20,14 +20,14 @@ task-type,task-name,task-group,cron-schedule,params
#
# Dataset quality scoring tasks
quality,quality-knb,metadig,0 0/1 * * * ?,"^eml.*|^http.*eml.*;knb.suite.1;urn:node:KNB;2020-08-24T00:00:00.00Z;1;1000"
-quality,quality-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;arctic.data.center.suite.1;urn:node:ARCTIC;2022-04-01T00:00:00.00Z;1;1000"
-quality,quality-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.3.1;urn:node:ARCTIC;2022-04-01T00:00:00.00Z;1;1000"
-quality,quality-dataone-fair,metadig,10 0/1 * * * ?,"^eml.*|^http.*eml.*|.*www.isotc211.org.*;FAIR-suite-0.3.1;urn:node:CN;2010-01-01T00:00:00.00Z;1;1000"
-quality,quality-ess-dive,metadig,15 0/1 * * * ?,"^eml.*|^http.*eml.*;ess-dive.data.center.suite-1.1.0;urn:node:ESS_DIVE;2016-01-01T00:00:00.00Z;1;1000;"
-quality,quality-cerp_sfwmd,metadig,20 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.3.1;urn:node:CERP_SFWMD;2010-10-07T00:00:00.00Z;1;1000;"
-quality,quality-CA_OPC,metadig,25 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.3.1;urn:node:CA_OPC;2010-10-07T00:00:00.00Z;1;1000;"
-quality,quality-sctld,metadig,30 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.3.1;urn:node:SCTLD;2010-10-07T00:00:00.00Z;1;1000;"
-quality,quality-drp,metadig,35 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.3.1;urn:node:DRP;2010-10-07T00:00:00.00Z;1;1000;"
+quality,quality-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;arctic.data.center.suite-1.2.0;urn:node:ARCTIC;2022-04-01T00:00:00.00Z;1;1000"
+quality,quality-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:ARCTIC;2022-04-01T00:00:00.00Z;1;1000"
+quality,quality-dataone-fair,metadig,10 0/1 * * * ?,"^eml.*|^http.*eml.*|.*www.isotc211.org.*;FAIR-suite-0.4.0;urn:node:CN;2010-01-01T00:00:00.00Z;1;1000"
+quality,quality-ess-dive,metadig,15 0/1 * * * ?,"^eml.*|^http.*eml.*;ess-dive.data.center.suite-1.2.0;urn:node:ESS_DIVE;2016-01-01T00:00:00.00Z;1;1000;"
+quality,quality-cerp_sfwmd,metadig,20 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:CERP_SFWMD;2010-10-07T00:00:00.00Z;1;1000;"
+quality,quality-CA_OPC,metadig,25 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:CA_OPC;2010-10-07T00:00:00.00Z;1;1000;"
+quality,quality-sctld,metadig,30 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:SCTLD;2010-10-07T00:00:00.00Z;1;1000;"
+quality,quality-drp,metadig,35 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:DRP;2010-10-07T00:00:00.00Z;1;1000;"
#
# Portal scoring tasks
score,portal-KNB-FAIR,metadig,1 0/1 * * * ?,"*portals*;FAIR-suite-0.3.1;urn:node:KNB;2020-08-10T00:00:00.00Z;1;100;portal"
diff --git a/helm/metadig-scheduler/values.yaml b/helm/metadig-scheduler/values.yaml
index 4d7e5ca5..8d49cb2d 100644
--- a/helm/metadig-scheduler/values.yaml
+++ b/helm/metadig-scheduler/values.yaml
@@ -7,7 +7,7 @@ replicaCount: 1
image:
repository: ghcr.io/nceas/metadig-scheduler
pullPolicy: Always
- tag: "v.2.5.0"
+ tag: "v.3.0.0"
imagePullSecrets: []
nameOverride: ""
diff --git a/helm/metadig-scorer/Chart.yaml b/helm/metadig-scorer/Chart.yaml
index e410cf2f..0ff7f758 100644
--- a/helm/metadig-scorer/Chart.yaml
+++ b/helm/metadig-scorer/Chart.yaml
@@ -15,10 +15,10 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.0.2
+version: 1.0.4
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
-appVersion: "2.5.0"
+appVersion: "3.0.0"
diff --git a/helm/metadig-scorer/values.yaml b/helm/metadig-scorer/values.yaml
index 7fc3eeca..3d6952df 100644
--- a/helm/metadig-scorer/values.yaml
+++ b/helm/metadig-scorer/values.yaml
@@ -7,7 +7,7 @@ replicaCount: 1
image:
repository: ghcr.io/nceas/metadig-scorer
pullPolicy: Always
- tag: "v.2.5.0"
+ tag: "v.3.0.0"
imagePullSecrets: []
diff --git a/helm/metadig-worker/Chart.yaml b/helm/metadig-worker/Chart.yaml
index 4ee19fad..18287cc9 100644
--- a/helm/metadig-worker/Chart.yaml
+++ b/helm/metadig-worker/Chart.yaml
@@ -15,10 +15,10 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.0.2
+version: 1.0.4
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
-appVersion: "2.5.0"
+appVersion: "3.0.0"
diff --git a/helm/metadig-worker/values.yaml b/helm/metadig-worker/values.yaml
index 2cbbb3cc..5c2bc8bd 100644
--- a/helm/metadig-worker/values.yaml
+++ b/helm/metadig-worker/values.yaml
@@ -7,7 +7,7 @@ replicaCount: 1
image:
repository: ghcr.io/nceas/metadig-worker
pullPolicy: Always
- tag: "v.2.5.0"
+ tag: "v.3.0.0"
imagePullSecrets: []
nameOverride: ""
diff --git a/pom.xml b/pom.xml
index 1a0503cf..cd942b78 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
4.0.0edu.ucsb.nceasmetadig-engine
- 2.5.0
+ 3.0.0jarmetadig-engineMetaDIG library for running metadata quality tests
@@ -16,14 +16,14 @@
0.8.25672.7.32.10.1
- 2.4.0
+ 2.8.02.7.0
- 2.20.0
+ 2.23.02.3.152.3.2
- 3.1.4.RELEASE
+ 5.3.22metadig2.5.0
@@ -39,6 +39,7 @@
http://nceas.ucsb.edu
+
@@ -63,9 +64,9 @@
- org.python
- jython-standalone
- ${jython.version}
+ black.ninia
+ jep
+ 4.1.1
@@ -96,7 +97,7 @@
commons-iocommons-io
- 2.13.0
+ 2.15.1org.apache.commons
@@ -182,7 +183,7 @@
org.apache.solrsolr-core
- 9.3.0
+ 9.4.1org.slf4j
@@ -217,7 +218,7 @@
org.slf4jslf4j-log4j12
- 1.6.1
+ 1.7.33log4j
@@ -243,7 +244,7 @@
commons-loggingcommons-logging
- 1.2
+ 1.3.0log4j
@@ -320,6 +321,10 @@
org.slf4jslf4j-api
+
+ org.springframework
+ spring-context-support
+
@@ -332,7 +337,7 @@
org.postgresqlpostgresql
- 42.6.0
+ 42.7.2
@@ -421,10 +426,10 @@
org.apache.maven.pluginsmaven-compiler-plugin
- 2.5.1
+ 3.8.1
- 1.8
- 1.8
+ 17
+ 17