Initial commit.

.dockerignore

@@ -0,0 +1,3 @@
+**/__pycache__
+**/*.egg_info
+**/*.pyc

.flake8

@@ -0,0 +1,6 @@
+[flake8]
+extend-exclude = conda_env/, working_dir/
+ignore = C901, E203, E266, E501, E722, E731, F401, F403, F405, W503
+max-line-length = 88
+max-complexity = 40
+select = B,C,E,F,W,T4,B9

.gitignore

@@ -0,0 +1,31 @@
+derby.log
+*.egg-info
+metastore_db
+__pycache__
+activate-env.bash
+build
+conda_env/
+dist
+scala_jar/data
+scala_jar/target
+scala_jar/project/target
+output_data/
+.ipynb_checkpoints
+.idea/
+*.class
+*.cache
+.DS_Store
+hlink/tests/output_data
+hlink/linking/reporting/output_reports
+hlink_config/derby
+hlink_config/spark_tmp_dir
+hlink_config/warehouse
+hlink_config/run.log
+*.pyc
+sphinx-docs/_*
+run_test.sh
+venv/
+working_dir/
+.coverage
+coverage_*
+*_coverage.xlsx

.pre-commit-config.yaml

@@ -0,0 +1,10 @@
+repos:
+- repo: https://github.com/ambv/black
+  rev: 20.8b1
+  hooks:
+  - id: black
+    language_version: python3.6
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v1.2.3
+  hooks:
+  - id: flake8

Dockerfile

@@ -0,0 +1,96 @@
+FROM python:3.6.5
+
+# --------------------- JAVA INSTALL START --------------------- #
+# Taken from the java openjdk 8 docker image: https://github.com/docker-library/openjdk/blob/7a33416016b60c045cf0ba99e82617ed6c130595/8/jre/slim/Dockerfile
+RUN apt-get update && apt-get install -y --no-install-recommends \
+		bzip2 \
+		unzip \
+		xz-utils \
+	&& rm -rf /var/lib/apt/lists/*
+
+# Default to UTF-8 file.encoding
+ENV LANG C.UTF-8
+
+# add a simple script that can auto-detect the appropriate JAVA_HOME value
+# based on whether the JDK or only the JRE is installed
+RUN { \
+		echo '#!/bin/sh'; \
+		echo 'set -e'; \
+		echo; \
+		echo 'dirname "$(dirname "$(readlink -f "$(which javac || which java)")")"'; \
+	} > /usr/local/bin/docker-java-home \
+	&& chmod +x /usr/local/bin/docker-java-home
+
+# do some fancy footwork to create a JAVA_HOME that's cross-architecture-safe
+RUN ln -svT "/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)" /docker-java-home
+ENV JAVA_HOME /docker-java-home/jre
+
+ENV JAVA_VERSION 8u212
+ENV JAVA_DEBIAN_VERSION 8u212-b01-1~deb9u1
+
+# see https://bugs.debian.org/775775
+# and https://github.com/docker-library/java/issues/19#issuecomment-70546872
+# ENV CA_CERTIFICATES_JAVA_VERSION 20170531+nmu1
+
+RUN set -ex; \
+	\
+# deal with slim variants not having man page directories (which causes "update-alternatives" to fail)
+	if [ ! -d /usr/share/man/man1 ]; then \
+		mkdir -p /usr/share/man/man1; \
+	fi; \
+	\
+	apt-get update; \
+	apt-get install -y --no-install-recommends \
+		openjdk-8-jre-headless \
+		# ca-certificates-java="$CA_CERTIFICATES_JAVA_VERSION" \
+	; \
+	rm -rf /var/lib/apt/lists/*; \
+	\
+# verify that "docker-java-home" returns what we expect
+	[ "$(readlink -f "$JAVA_HOME")" = "$(docker-java-home)" ]; \
+	\
+# update-alternatives so that future installs of other OpenJDK versions don't change /usr/bin/java
+	update-alternatives --get-selections | awk -v home="$(readlink -f "$JAVA_HOME")" 'index($3, home) == 1 { $2 = "manual"; print | "update-alternatives --set-selections" }'; \
+# ... and verify that it actually worked for one of the alternatives we care about
+	update-alternatives --query java | grep -q 'Status: manual'
+
+# see CA_CERTIFICATES_JAVA_VERSION notes above
+RUN /var/lib/dpkg/info/ca-certificates-java.postinst configure
+
+# --------------------- JAVA INSTALL END --------------------- #
+
+# --------------------- SBT INSTALL --------------------- #
+ENV SCALA_VERSION 2.11.12
+ENV SBT_VERSION 1.1.6
+
+# Scala expects this file
+RUN touch /usr/lib/jvm/java-8-openjdk-amd64/release
+
+# Install Scala
+## Piping curl directly in tar
+RUN \
+  curl -fsL https://downloads.typesafe.com/scala/$SCALA_VERSION/scala-$SCALA_VERSION.tgz | tar xfz - -C /root/ && \
+  echo >> /root/.bashrc && \
+  echo "export PATH=~/scala-$SCALA_VERSION/bin:$PATH" >> /root/.bashrc
+
+# Install sbt
+RUN \
+  curl -L -o sbt-$SBT_VERSION.deb "https://scala.jfrog.io/ui/api/v1/download?repoKey=debian&path=/sbt-$SBT_VERSION.deb" && \
+  dpkg -i sbt-$SBT_VERSION.deb && \
+  rm sbt-$SBT_VERSION.deb && \
+  apt-get update && \
+  apt-get install sbt && \
+  sbt sbtVersion
+
+# --------------------- SBT INSTALL END --------------------- #
+
+
+RUN mkdir /hlink
+WORKDIR /hlink
+
+COPY scala_jar scala_jar
+RUN cd scala_jar && sbt assembly 
+
+COPY . .
+RUN mv scala_jar/target/scala-2.11/*.jar hlink/spark/jars/
+RUN pip install .

Jenkinsfile

@@ -0,0 +1,36 @@
+node {
+	def image_id = "hlink:${env.BUILD_TAG}"
+	stage("Checkout") {
+		deleteDir()
+		checkout scm
+	}
+	stage("Deploy") {
+		host = "gp1.pop.umn.edu"
+		deploy_target = "/pkg/ipums/programming/linking/hlink/deploys/${env.BRANCH_NAME}"      
+		def target_exists = sh script: "ssh ${host} 'test -d ${deploy_target}'", returnStatus:true
+		if (target_exists != 0) {
+			sh "ssh ${host} 'mkdir ${deploy_target} && cd ${deploy_target} && mkdir scripts'"
+			sh "ssh ${host} 'cd ${deploy_target} && git clone [email protected]:mpc/hlink.git'"
+			sh "ssh ${host} 'cd ${deploy_target} && /pkg/ipums/programming/conda/v4.8/envs/hlink/bin/virtualenv -p 3.6.5 venv'"
+		}				
+		sh "ssh ${host} 'cd ${deploy_target}/hlink && git checkout ${env.BRANCH_NAME} && git pull origin ${env.BRANCH_NAME}'"
+		sh "rsync -av ./deploy/hlink ${host}:${deploy_target}/scripts/hlink"
+		sh "rsync -av ./deploy/global_conf.json ${host}:${deploy_target}/global_conf.json"
+		sh "ssh ${host} 'cd ${deploy_target} && sed -i \'s/XXX_BRANCH/${env.BRANCH_NAME}/g\' global_conf.json && sed -i \'s/XXX_BRANCH/${env.BRANCH_NAME}/g\' scripts/hlink'"
+		//sh "ssh ${host} 'cd ${deploy_target}/hlink/scala_jar && rm -rf target && /pkg/mpctools/bin/sbt assembly && cp ./target/scala-2.11/hlink_lib-assembly-1.0.jar ../hlink/spark/jars'"            
+		sh "ssh ${host} 'cd ${deploy_target} && venv/bin/pip install ./hlink'"
+  }
+
+	/*stage("Build") {
+		docker.build(image_id)
+	}
+	stage("Black") {
+		sh "docker run ${image_id} black --check ."
+	}
+	stage("Flake8") {
+		sh "docker run ${image_id} flake8 --count ."
+	}
+	stage("Test") {
+		sh "docker run ${image_id} pytest hlink/tests/"
+	}*/
+}