Improve annotator installation scripts

.dockerignore

@@ -0,0 +1,17 @@
+python/python/bystro/*.so
+python/target
+python/build
+python/python/bystro/examples
+python/python/bystro/ancestry/data
+**/*.ipynb
+**/.mypy_cache
+**/.coverage
+**/.ipynb_checkpoints
+**/*.log
+**/__pycache__
+perl/.build
+perl/.build_backup/
+perl/*tar.gz
+perl/.vscode
+perl/.tidyall.d
+perl/Bystro-*

.github/workflows/perl-build-docker.yml

@@ -0,0 +1,40 @@
+name: (Annotator) Build and Run Annotator Perl Dockerfile
+
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build Docker Image from Local Context
+        run: |
+          docker build -t bystro-annotator -f Dockerfile.perl .
+
+      - name: Run Docker Image and Capture Help Output
+        continue-on-error: true # This is necessary because the help output returns a 255
+        id: help-output
+        run: |
+          docker run --rm bystro-annotator &> help_output.txt
+
+      - name: Assert Help Output Starts with Expected String
+        run: |
+          first_line=$(head -n 1 help_output.txt)
+          if [[ "$first_line" == "usage: bystro-annotate.pl"* ]]; then
+            echo "Help output assertion passed!"
+          else
+            echo "Help output assertion failed. Got: $first_line"
+            exit 1
+          fi
+
+      - name: Upload Help Output as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: help-output
+          path: help_output.txt

.github/workflows/perl-code-tidy.yml

@@ -3,14 +3,14 @@ name: Perl Tidy & Critic Check
 on:
   push:
     paths:
-    - '**.pl'
-    - '**.pm'
-    - '**.t'
+      - "**.pl"
+      - "**.pm"
+      - "**.t"
   pull_request:
     paths:
-    - '**.pl'
-    - '**.pm'
-    - '**.t'
+      - "**.pl"
+      - "**.pm"
+      - "**.t"
 
 jobs:
   tidy-critic-check:

.github/workflows/perl-package-build.yml

@@ -22,85 +22,84 @@ jobs:
       - name: Install Go
         shell: bash
         run: |
-            set -eux; \
-            arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
-            url=; \
-            case "$arch" in \
-              'amd64') \
-                url='https://dl.google.com/go/go1.21.3.linux-amd64.tar.gz'; \
-                sha256='1241381b2843fae5a9707eec1f8fb2ef94d827990582c7c7c32f5bdfbfd420c8'; \
-                ;; \
-              'armel') \
-                export GOARCH='arm' GOARM='5' GOOS='linux'; \
-                ;; \
-              'armhf') \
-                url='https://dl.google.com/go/go1.21.3.linux-armv6l.tar.gz'; \
-                sha256='a1ddcaaf0821a12a800884c14cb4268ce1c1f5a0301e9060646f1e15e611c6c7'; \
-                ;; \
-              'arm64') \
-                url='https://dl.google.com/go/go1.21.3.linux-arm64.tar.gz'; \
-                sha256='fc90fa48ae97ba6368eecb914343590bbb61b388089510d0c56c2dde52987ef3'; \
-                ;; \
-              'i386') \
-                url='https://dl.google.com/go/go1.21.3.linux-386.tar.gz'; \
-                sha256='fb209fd070db500a84291c5a95251cceeb1723e8f6142de9baca5af70a927c0e'; \
-                ;; \
-              'mips64el') \
-                url='https://dl.google.com/go/go1.21.3.linux-mips64le.tar.gz'; \
-                sha256='a569ffbc88b4e14cf2682f65cec950460665e4392b0d78b8868b4718c979bda8'; \
-                ;; \
-              'ppc64el') \
-                url='https://dl.google.com/go/go1.21.3.linux-ppc64le.tar.gz'; \
-                sha256='3b0e10a3704f164a6e85e0377728ec5fd21524fabe4c925610e34076586d5826'; \
-                ;; \
-              'riscv64') \
-                url='https://dl.google.com/go/go1.21.3.linux-riscv64.tar.gz'; \
-                sha256='67d14d3e513e505d1ec3ea34b55641c6c29556603c7899af94045c170c1c0f94'; \
-                ;; \
-              's390x') \
-                url='https://dl.google.com/go/go1.21.3.linux-s390x.tar.gz'; \
-                sha256='4c78e2e6f4c684a3d5a9bdc97202729053f44eb7be188206f0627ef3e18716b6'; \
-                ;; \
-              *) echo >&2 "error: unsupported architecture '$arch' (likely packaging update needed)"; exit 1 ;; \
-            esac; \
-            build=; \
-            if [ -z "$url" ]; then \
-            # https://github.com/golang/go/issues/38536#issuecomment-616897960
-              build=1; \
-              url='https://dl.google.com/go/go1.21.3.src.tar.gz'; \
-              sha256='186f2b6f8c8b704e696821b09ab2041a5c1ee13dcbc3156a13adcf75931ee488'; \
-              echo >&2; \
-              echo >&2 "warning: current architecture ($arch) does not have a compatible Go binary release; will be building from source"; \
-              echo >&2; \
-            fi; \
-            \
-            wget -O go.tgz.asc "$url.asc"; \
-            wget -O go.tgz "$url" --progress=dot:giga; \
-            echo "$sha256 *go.tgz" | sha256sum -c -; \
-            \
-            # https://github.com/golang/go/issues/14739#issuecomment-324767697
-            GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \
-            # https://www.google.com/linuxrepositories/
-            gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 'EB4C 1BFD 4F04 2F6D DDCC  EC91 7721 F63B D38B 4796'; \
-            # let's also fetch the specific subkey of that key explicitly that we expect "go.tgz.asc" to be signed by, just to make sure we definitely have it
-            gpg --batch --keyserver keyserver.ubuntu.com --recv-keys '2F52 8D36 D67B 69ED F998  D857 78BD 6547 3CB3 BD13'; \
-            gpg --batch --verify go.tgz.asc go.tgz; \
-            gpgconf --kill all; \
-            rm -rf "$GNUPGHOME" go.tgz.asc; \
-            \
-            tar -C /usr/local -xzf go.tgz; \
-            rm go.tgz;
-            /usr/local/go/bin/go version
-            echo "/usr/local/go/bin" >> $GITHUB_PATH
-            echo "${HOME}/go/bin" >> $GITHUB_PATH
+          set -eux; \
+          arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
+          url=; \
+          case "$arch" in \
+            'amd64') \
+              url='https://dl.google.com/go/go1.21.3.linux-amd64.tar.gz'; \
+              sha256='1241381b2843fae5a9707eec1f8fb2ef94d827990582c7c7c32f5bdfbfd420c8'; \
+              ;; \
+            'armel') \
+              export GOARCH='arm' GOARM='5' GOOS='linux'; \
+              ;; \
+            'armhf') \
+              url='https://dl.google.com/go/go1.21.3.linux-armv6l.tar.gz'; \
+              sha256='a1ddcaaf0821a12a800884c14cb4268ce1c1f5a0301e9060646f1e15e611c6c7'; \
+              ;; \
+            'arm64') \
+              url='https://dl.google.com/go/go1.21.3.linux-arm64.tar.gz'; \
+              sha256='fc90fa48ae97ba6368eecb914343590bbb61b388089510d0c56c2dde52987ef3'; \
+              ;; \
+            'i386') \
+              url='https://dl.google.com/go/go1.21.3.linux-386.tar.gz'; \
+              sha256='fb209fd070db500a84291c5a95251cceeb1723e8f6142de9baca5af70a927c0e'; \
+              ;; \
+            'mips64el') \
+              url='https://dl.google.com/go/go1.21.3.linux-mips64le.tar.gz'; \
+              sha256='a569ffbc88b4e14cf2682f65cec950460665e4392b0d78b8868b4718c979bda8'; \
+              ;; \
+            'ppc64el') \
+              url='https://dl.google.com/go/go1.21.3.linux-ppc64le.tar.gz'; \
+              sha256='3b0e10a3704f164a6e85e0377728ec5fd21524fabe4c925610e34076586d5826'; \
+              ;; \
+            'riscv64') \
+              url='https://dl.google.com/go/go1.21.3.linux-riscv64.tar.gz'; \
+              sha256='67d14d3e513e505d1ec3ea34b55641c6c29556603c7899af94045c170c1c0f94'; \
+              ;; \
+            's390x') \
+              url='https://dl.google.com/go/go1.21.3.linux-s390x.tar.gz'; \
+              sha256='4c78e2e6f4c684a3d5a9bdc97202729053f44eb7be188206f0627ef3e18716b6'; \
+              ;; \
+            *) echo >&2 "error: unsupported architecture '$arch' (likely packaging update needed)"; exit 1 ;; \
+          esac; \
+          build=; \
+          if [ -z "$url" ]; then \
+          # https://github.com/golang/go/issues/38536#issuecomment-616897960
+            build=1; \
+            url='https://dl.google.com/go/go1.21.3.src.tar.gz'; \
+            sha256='186f2b6f8c8b704e696821b09ab2041a5c1ee13dcbc3156a13adcf75931ee488'; \
+            echo >&2; \
+            echo >&2 "warning: current architecture ($arch) does not have a compatible Go binary release; will be building from source"; \
+            echo >&2; \
+          fi; \
+          \
+          wget -O go.tgz.asc "$url.asc"; \
+          wget -O go.tgz "$url" --progress=dot:giga; \
+          echo "$sha256 *go.tgz" | sha256sum -c -; \
+          \
+          # https://github.com/golang/go/issues/14739#issuecomment-324767697
+          GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \
+          # https://www.google.com/linuxrepositories/
+          gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 'EB4C 1BFD 4F04 2F6D DDCC  EC91 7721 F63B D38B 4796'; \
+          # let's also fetch the specific subkey of that key explicitly that we expect "go.tgz.asc" to be signed by, just to make sure we definitely have it
+          gpg --batch --keyserver keyserver.ubuntu.com --recv-keys '2F52 8D36 D67B 69ED F998  D857 78BD 6547 3CB3 BD13'; \
+          gpg --batch --verify go.tgz.asc go.tgz; \
+          gpgconf --kill all; \
+          rm -rf "$GNUPGHOME" go.tgz.asc; \
+          \
+          tar -C /usr/local -xzf go.tgz; \
+          rm go.tgz;
+          /usr/local/go/bin/go version
+          echo "/usr/local/go/bin" >> $GITHUB_PATH
+          echo "${HOME}/go/bin" >> $GITHUB_PATH
 
       - name: Install one-off Bystro dependencies
         shell: bash
         run: |
           go install github.com/bystrogenomics/[email protected]
           cpm install -g https://github.com/bystrogenomics/msgpack-perl.git
           cpm install -g --no-test MouseX::Getopt
-          cpm install -g DBD::[email protected]
           git clone --depth 1 --recurse-submodules https://github.com/salortiz/LMDB_File.git \
             && cd LMDB_File \
             && cpanm --quiet .

.github/workflows/perl-test-install-apt.yml

@@ -0,0 +1,118 @@
+name: Install and Test Bystro (APT)
+
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Checkout the repository
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Run the install script using APT
+      - name: Install Bystro using APT script
+        run: |
+          touch ~/.profile
+          ./install-apt.sh --profile-file ~/.profile
+
+      # Test that bystro-annotate.pl works
+      # When it runs with --help it exits with 255
+      - name: Verify installation by running bystro-annotate.pl --help
+        continue-on-error: true
+        run: |
+          source ~/.profile
+          output=$(bystro-annotate.pl --help 2>&1)
+          exit_code=$?
+          first_line=$(echo "$output" | head -n 1)
+
+          if [ "$exit_code" -eq 255 ]; then
+            echo "Exit code 255 confirmed."
+          else
+            echo "Unexpected exit code: $exit_code" >&2
+            exit 1
+          fi
+
+          if [[ "$first_line" == "usage: bystro-annotate.pl"* ]]; then
+            echo "First line matches the expected 'usage: bystro-annotate.pl'."
+          else
+            echo "First line does not match the expected 'usage: bystro-annotate.pl'." >&2
+            echo "Actual first line: $first_line" >&2
+            exit 1
+          fi
+
+      # Test that bystro-build.pl works
+      # When it runs with --help it exits with 1
+      - name: Verify installation by running bystro-build.pl --help
+        continue-on-error: true
+        run: |
+          source ~/.profile
+          output=$(bystro-build.pl --help 2>&1)
+          first_line=$(echo "$output" | head -n 1)
+
+          if [[ "$first_line" == "Usage:"* ]]; then
+            echo "First line matches the expected 'Usage:'"
+          else
+            echo "First line does not match the expected 'Usage:'" >&2
+            echo "Actual first line: $first_line" >&2
+            exit 1
+          fi
+
+      - name: Run tests
+        run: |
+          cd perl
+          source ~/.profile
+          prove -r -j$(nproc) t
+
+      - name: Check that expected programs are in the PATH
+        run: |
+          source ~/.profile
+          if command -v bystro-annotate.pl > /dev/null; then
+            echo "bystro-annotate.pl is in the PATH"
+          else
+            echo "bystro-annotate.pl is not in the PATH" >&2
+            exit 1
+          fi
+
+          if command -v bystro-build.pl > /dev/null; then
+            echo "bystro-build.pl is in the PATH"
+          else
+            echo "bystro-build.pl is not in the PATH" >&2
+            exit 1
+          fi
+
+          if command -v bystro-vcf > /dev/null; then
+            echo "bystro-vcf is in the PATH"
+          else
+            echo "bystro-vcf is not in the PATH" >&2
+            exit 1
+          fi
+
+          if command -v bystro-snp > /dev/null; then
+            echo "bystro-snp is in the PATH"
+          else
+            echo "bystro-snp is not in the PATH" >&2
+            exit 1
+          fi
+
+          if command -v bystro-stats > /dev/null; then
+            echo "bystro-stats is in the PATH"
+          else
+            echo "bystro-stats is not in the PATH" >&2
+            exit 1
+          fi
+
+          if command -v bgzip > /dev/null; then
+            echo "bgzip is in the PATH"
+          else
+            echo "bgzip is not in the PATH" >&2
+            exit 1
+          fi
+
+          if command -v yq > /dev/null; then
+            echo "yq is in the PATH"
+          else
+            echo "yq is not in the PATH" >&2
+            exit 1
+          fi