diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0ee67495..a0543487 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,6 +35,23 @@ jobs: - name: Install linux deps run: sudo apt-get install -y libcurl4-openssl-dev + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install python dependencies + run: pip install jep==3.9 # Install the jep library + + - name: Set JEP_LIBRARY_PATH environment variable + run: | + echo "JEP_LIBRARY_PATH=$(python -c 'import site; print(site.getsitepackages()[0])')/jep" >> $GITHUB_ENV + + - name: test + run: | + echo $JEP_LIBRARY_PATH + ls $JEP_LIBRARY_PATH + - name: Install deps run: Rscript -e 'install.packages(c("httr", "jsonlite", "remotes"))' @@ -58,7 +75,7 @@ jobs: docker-publish: name: Docker Build and Publish - if: github.ref_name == 'develop' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref_name, 'bugfix') + if: github.ref_name == 'develop' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref_name, 'feature') needs: maven-build runs-on: ubuntu-latest permissions: diff --git a/.vscode/launch.json b/.vscode/launch.json index af81a530..3bb1538e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -20,6 +20,18 @@ ".", "${fileDirname}" ] + }, + { + "name": "Run MDQEngine", + "type": "java", + "request": "launch", + "mainClass": "edu.ucsb.nceas.mdqengine.MDQEngine", + "args": [ + "/opt/local/metadig/suites/python-suite.xml", + "${workspaceFolder}/src/test/resources/test-docs/doi:10.18739_A2W08WG3R.xml", + "${workspaceFolder}/src/test/resources/test-docs/doi:10.18739_A2W08WG3R.sm", + ], + "cwd": "${workspaceFolder}" } ] } \ No newline at end of file diff --git a/Docker/metadig-worker/Dockerfile b/Docker/metadig-worker/Dockerfile index 2fac2518..b1a24c53 100644 --- a/Docker/metadig-worker/Dockerfile +++ b/Docker/metadig-worker/Dockerfile @@ -11,7 +11,7 @@ LABEL maintainer="Jeanette Clark jclark@nceas.ucsb.edu" # Set the working directory WORKDIR /var/lib/metadig -ARG ENGINE_TAG=2.0.0 +ARG ENGINE_TAG=3.0.0-SNAPSHOT ENV ENGINE_TAG=${ENGINE_TAG} # The most recently built jar file is copied from the maven build directory to this dir by maven, so that @@ -24,7 +24,11 @@ COPY target/metadig-engine-$ENGINE_TAG.jar metadig-engine-$ENGINE_TAG.jar COPY Docker/metadig-worker/solr solr/ RUN apt-get update -RUN apt-get install -y g++ r-base r-base-dev r-base-core r-doc-html libc6-dev libssl-dev libxml2 libxml2-dev libcurl4-openssl-dev +RUN apt-get install -y g++ git r-base r-base-dev r-base-core r-doc-html libc6-dev libssl-dev libxml2 libxml2-dev libcurl4-openssl-dev python3 python3-pip python3-dev openjdk-17-jdk + +# Set Java environment variables +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 +ENV PATH="$PATH:$JAVA_HOME/bin" # Install required R packages RUN echo 'options(repos = c(CRAN = "http://cran.rstudio.com"))' >> /usr/lib/R/etc/Rprofile.site @@ -34,6 +38,9 @@ RUN Rscript -e "remotes::install_github('NCEAS/metadig-r', ref = 'v.0.2.0')" # DataONE indexer prints copious error msgs if these files don't exist RUN mkdir -p /etc/dataone/index && touch /etc/dataone/index/d1client.properties && touch /etc/dataone/node.properties && touch /etc/dataone/index/cn-index-processor.properties +COPY Docker/metadig-worker/requirements.txt /opt/local/metadig/ +RUN pip install -r /opt/local/metadig/requirements.txt + # Run the Worker process # Note: docker --buile-arg only allows one argument (one token only, multiple tokens inside quotes doesn't work, so have # to specify java options directly on command line. diff --git a/Docker/metadig-worker/requirements.txt b/Docker/metadig-worker/requirements.txt new file mode 100644 index 00000000..5186f3f6 --- /dev/null +++ b/Docker/metadig-worker/requirements.txt @@ -0,0 +1,2 @@ +numpy +metadig @ git+https://github.com/NCEAS/metadig-py.git diff --git a/bin/sendAssessmentTest.py b/bin/sendAssessmentTest.py index 7f8ecc36..8e0cc543 100755 --- a/bin/sendAssessmentTest.py +++ b/bin/sendAssessmentTest.py @@ -1,15 +1,12 @@ -#!/Users/clark/.virtualenvs/metadig/bin/python - -# this is a shebang to your python path. needs to be 2.7 -#/usr/bin/python +#!/usr/bin/python # This Python program can be used for local testing of RabbitMQ messaging between # metadig services, assessment processing and Solr indexing. import sys import re -print "Sending assessment request to metadig-controler (test mode):" -print "Number of arguments: ", len(sys.argv) +print("Sending assessment request to metadig-controller (test mode):") +print("Number of arguments: ", len(sys.argv)) import socket host="localhost" @@ -24,7 +21,7 @@ clientsocket.connect((host, portNum)) testCount = 1 -suiteId = "test.suite" +suiteId = "python.suite" nodeId = "urn:node:ARCTIC" idVal = "doi:10.18739/A2W08WG3R" @@ -34,17 +31,17 @@ smFile = "%s/%s.sm" % (testDir, fileIdVal) # First send the type of request, either 'graph' or 'quality' -clientsocket.send("%s\n" % 'quality') +clientsocket.send('quality\n'.encode('utf-8')) # Next send the number of tests that will be run -clientsocket.send("%s\n" % testCount) +clientsocket.send(f"{testCount}\n".encode('utf-8')) msg = '%s,%s,%s,%s,%s\n' % (idVal, mdFile, smFile, suiteId, nodeId) print(msg) -clientsocket.send(msg) +clientsocket.send(msg.encode('utf-8')) # Server will stop reading from the current port connection -clientsocket.send("Done\n") +clientsocket.send("Done\n".encode('utf-8')) # Server will stop reading from any port connection, but will continue to run clientsocket.close() diff --git a/bin/startController.sh b/bin/startController.sh index d7064d32..23eb518e 100755 --- a/bin/startController.sh +++ b/bin/startController.sh @@ -1,6 +1,6 @@ #!/bin/bash -version=2.5.0 +version=3.0.0-SNAPSHOT # Use first argument if sending jobs via port, for testing, for example: java -cp /opt/local/metadig/config:./target/metadig-engine-${version}.jar edu.ucsb.nceas.mdqengine.Controller 33000 diff --git a/bin/startWorker.sh b/bin/startWorker.sh index a5c7da61..640ff99c 100755 --- a/bin/startWorker.sh +++ b/bin/startWorker.sh @@ -1,10 +1,7 @@ #!/bin/bash # Start a local copy of metadig-worker for RabbitMQ based development testing. -version=2.5.0 +version=3.0.0-SNAPSHOT -# Include the MetaDIG python library -export JYTHONPATH=/opt/local/metadig/metadig-py - -java -cp /opt/local/metadig/config:./target/metadig-engine-${version}.jar:./target/classes/solr -Dpython.path=JYTHONPATH edu.ucsb.nceas.mdqengine.Worker +java -cp /opt/local/metadig/config:./target/metadig-engine-${version}.jar:./target/classes/solr edu.ucsb.nceas.mdqengine.Worker diff --git a/helm/metadig-controller/Chart.yaml b/helm/metadig-controller/Chart.yaml index 564e1d8d..a4eebb89 100644 --- a/helm/metadig-controller/Chart.yaml +++ b/helm/metadig-controller/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.2 +version: 1.0.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2.5.0" +appVersion: "3.0.0" diff --git a/helm/metadig-controller/config.dev/metadig.properties b/helm/metadig-controller/config.dev/metadig.properties index ada5d878..added0b1 100644 --- a/helm/metadig-controller/config.dev/metadig.properties +++ b/helm/metadig-controller/config.dev/metadig.properties @@ -33,4 +33,6 @@ quartz.monitor = true quartz.monitor.schedule = 0 0 0 * * ? quartz.monitor.processing.time = 12 quartz.monitor.run.limit = 10 +# jep library +jep.path = /usr/local/lib/python3.10/dist-packages/jep diff --git a/helm/metadig-controller/config/metadig.properties b/helm/metadig-controller/config/metadig.properties index 3674dda1..23044e29 100644 --- a/helm/metadig-controller/config/metadig.properties +++ b/helm/metadig-controller/config/metadig.properties @@ -59,3 +59,5 @@ quartz.monitor = true quartz.monitor.schedule = 0 0 0 * * ? quartz.monitor.processing.time = 12 quartz.monitor.run.limit = 10 +# jep library +jep.path = /usr/local/lib/python3.10/dist-packages/jep diff --git a/helm/metadig-controller/values.yaml b/helm/metadig-controller/values.yaml index 80931bc7..0643279d 100644 --- a/helm/metadig-controller/values.yaml +++ b/helm/metadig-controller/values.yaml @@ -7,7 +7,7 @@ replicaCount: 1 image: repository: ghcr.io/nceas/metadig-controller pullPolicy: Always - tag: "v.2.5.0" + tag: "v.3.0.0" imagePullSecrets: [] diff --git a/helm/metadig-scheduler/Chart.yaml b/helm/metadig-scheduler/Chart.yaml index 9ae738fa..745af7e7 100644 --- a/helm/metadig-scheduler/Chart.yaml +++ b/helm/metadig-scheduler/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.3 +version: 1.0.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2.5.0" +appVersion: "3.0.0" diff --git a/helm/metadig-scheduler/config.dev/taskList.csv b/helm/metadig-scheduler/config.dev/taskList.csv index 84b89c2e..86e64ed7 100644 --- a/helm/metadig-scheduler/config.dev/taskList.csv +++ b/helm/metadig-scheduler/config.dev/taskList.csv @@ -19,8 +19,8 @@ task-type,task-name,task-group,cron-schedule,params # - requestType: for score tasks, determine type of portal processing ("portal" or "node") # # Dataset quality scoring tasks -quality,quality-test-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;arctic.data.center.suite.1;urn:node:mnTestARCTIC;2022-05-01T00:00:00.00Z;1;1000" -quality,quality-test-dataone-fair,metadig,10 0/1 * * * ?,"^eml.*|^http.*eml.*|.*www.isotc211.org.*;FAIR-suite-0.3.1;urn:node:cnStage;2022-05-01T00:00:00.00Z;1;1000" +quality,quality-test-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;arctic.data.center.suite-1.2.0;urn:node:mnTestARCTIC;2022-05-01T00:00:00.00Z;1;1000" +quality,quality-test-dataone-fair,metadig,10 0/1 * * * ?,"^eml.*|^http.*eml.*|.*www.isotc211.org.*;FAIR-suite-0.4.0;urn:node:cnStage;2022-05-01T00:00:00.00Z;1;1000" # # Portal scoring tasks score,portal-test-arctic-FAIR,metadig,10 0/1 * * * ?,"*portals*;FAIR-suite-0.3.1;urn:node:mnTestARCTIC;2022-05-01T00:00:00.00Z;1;100;portal" diff --git a/helm/metadig-scheduler/config/taskList.csv b/helm/metadig-scheduler/config/taskList.csv index 70a5acbf..989c936b 100644 --- a/helm/metadig-scheduler/config/taskList.csv +++ b/helm/metadig-scheduler/config/taskList.csv @@ -20,14 +20,14 @@ task-type,task-name,task-group,cron-schedule,params # # Dataset quality scoring tasks quality,quality-knb,metadig,0 0/1 * * * ?,"^eml.*|^http.*eml.*;knb.suite.1;urn:node:KNB;2020-08-24T00:00:00.00Z;1;1000" -quality,quality-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;arctic.data.center.suite.1;urn:node:ARCTIC;2022-04-01T00:00:00.00Z;1;1000" -quality,quality-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.3.1;urn:node:ARCTIC;2022-04-01T00:00:00.00Z;1;1000" -quality,quality-dataone-fair,metadig,10 0/1 * * * ?,"^eml.*|^http.*eml.*|.*www.isotc211.org.*;FAIR-suite-0.3.1;urn:node:CN;2010-01-01T00:00:00.00Z;1;1000" -quality,quality-ess-dive,metadig,15 0/1 * * * ?,"^eml.*|^http.*eml.*;ess-dive.data.center.suite-1.1.0;urn:node:ESS_DIVE;2016-01-01T00:00:00.00Z;1;1000;" -quality,quality-cerp_sfwmd,metadig,20 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.3.1;urn:node:CERP_SFWMD;2010-10-07T00:00:00.00Z;1;1000;" -quality,quality-CA_OPC,metadig,25 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.3.1;urn:node:CA_OPC;2010-10-07T00:00:00.00Z;1;1000;" -quality,quality-sctld,metadig,30 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.3.1;urn:node:SCTLD;2010-10-07T00:00:00.00Z;1;1000;" -quality,quality-drp,metadig,35 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.3.1;urn:node:DRP;2010-10-07T00:00:00.00Z;1;1000;" +quality,quality-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;arctic.data.center.suite-1.2.0;urn:node:ARCTIC;2022-04-01T00:00:00.00Z;1;1000" +quality,quality-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:ARCTIC;2022-04-01T00:00:00.00Z;1;1000" +quality,quality-dataone-fair,metadig,10 0/1 * * * ?,"^eml.*|^http.*eml.*|.*www.isotc211.org.*;FAIR-suite-0.4.0;urn:node:CN;2010-01-01T00:00:00.00Z;1;1000" +quality,quality-ess-dive,metadig,15 0/1 * * * ?,"^eml.*|^http.*eml.*;ess-dive.data.center.suite-1.2.0;urn:node:ESS_DIVE;2016-01-01T00:00:00.00Z;1;1000;" +quality,quality-cerp_sfwmd,metadig,20 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:CERP_SFWMD;2010-10-07T00:00:00.00Z;1;1000;" +quality,quality-CA_OPC,metadig,25 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:CA_OPC;2010-10-07T00:00:00.00Z;1;1000;" +quality,quality-sctld,metadig,30 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:SCTLD;2010-10-07T00:00:00.00Z;1;1000;" +quality,quality-drp,metadig,35 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:DRP;2010-10-07T00:00:00.00Z;1;1000;" # # Portal scoring tasks score,portal-KNB-FAIR,metadig,1 0/1 * * * ?,"*portals*;FAIR-suite-0.3.1;urn:node:KNB;2020-08-10T00:00:00.00Z;1;100;portal" diff --git a/helm/metadig-scheduler/values.yaml b/helm/metadig-scheduler/values.yaml index 4d7e5ca5..8d49cb2d 100644 --- a/helm/metadig-scheduler/values.yaml +++ b/helm/metadig-scheduler/values.yaml @@ -7,7 +7,7 @@ replicaCount: 1 image: repository: ghcr.io/nceas/metadig-scheduler pullPolicy: Always - tag: "v.2.5.0" + tag: "v.3.0.0" imagePullSecrets: [] nameOverride: "" diff --git a/helm/metadig-scorer/Chart.yaml b/helm/metadig-scorer/Chart.yaml index e410cf2f..0ff7f758 100644 --- a/helm/metadig-scorer/Chart.yaml +++ b/helm/metadig-scorer/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.2 +version: 1.0.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2.5.0" +appVersion: "3.0.0" diff --git a/helm/metadig-scorer/values.yaml b/helm/metadig-scorer/values.yaml index 7fc3eeca..3d6952df 100644 --- a/helm/metadig-scorer/values.yaml +++ b/helm/metadig-scorer/values.yaml @@ -7,7 +7,7 @@ replicaCount: 1 image: repository: ghcr.io/nceas/metadig-scorer pullPolicy: Always - tag: "v.2.5.0" + tag: "v.3.0.0" imagePullSecrets: [] diff --git a/helm/metadig-worker/Chart.yaml b/helm/metadig-worker/Chart.yaml index 4ee19fad..18287cc9 100644 --- a/helm/metadig-worker/Chart.yaml +++ b/helm/metadig-worker/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.2 +version: 1.0.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2.5.0" +appVersion: "3.0.0" diff --git a/helm/metadig-worker/values.yaml b/helm/metadig-worker/values.yaml index 2cbbb3cc..5c2bc8bd 100644 --- a/helm/metadig-worker/values.yaml +++ b/helm/metadig-worker/values.yaml @@ -7,7 +7,7 @@ replicaCount: 1 image: repository: ghcr.io/nceas/metadig-worker pullPolicy: Always - tag: "v.2.5.0" + tag: "v.3.0.0" imagePullSecrets: [] nameOverride: "" diff --git a/pom.xml b/pom.xml index 1a0503cf..cd942b78 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 edu.ucsb.nceas metadig-engine - 2.5.0 + 3.0.0 jar metadig-engine MetaDIG library for running metadata quality tests @@ -16,14 +16,14 @@ 0.8.2567 2.7.3 2.10.1 - 2.4.0 + 2.8.0 2.7.0 - 2.20.0 + 2.23.0 2.3.15 2.3.2 - 3.1.4.RELEASE + 5.3.22 metadig 2.5.0 @@ -39,6 +39,7 @@ http://nceas.ucsb.edu + @@ -63,9 +64,9 @@ - org.python - jython-standalone - ${jython.version} + black.ninia + jep + 4.1.1 @@ -96,7 +97,7 @@ commons-io commons-io - 2.13.0 + 2.15.1 org.apache.commons @@ -182,7 +183,7 @@ org.apache.solr solr-core - 9.3.0 + 9.4.1 org.slf4j @@ -217,7 +218,7 @@ org.slf4j slf4j-log4j12 - 1.6.1 + 1.7.33 log4j @@ -243,7 +244,7 @@ commons-logging commons-logging - 1.2 + 1.3.0 log4j @@ -320,6 +321,10 @@ org.slf4j slf4j-api + + org.springframework + spring-context-support + @@ -332,7 +337,7 @@ org.postgresql postgresql - 42.6.0 + 42.7.2 @@ -421,10 +426,10 @@ org.apache.maven.plugins maven-compiler-plugin - 2.5.1 + 3.8.1 - 1.8 - 1.8 + 17 + 17