Dockerfile and docker-image.yml (github action) (#57)

* Dockerfile and docker-image.yml (github action) * Dockerfile squash (#56) The github actions didn't start, trying with the new branch. * Add documentation for Docker images. * Update Dockerfile * Add pipeline_navvis_rig. * fixing mistake from merge * fixing mistake from merge #2 * fixing mistake from merge #3 * Create a temporal folder for the QR code detection output (useful for docker image, otherwise it fails since it can't create the folder because it doesn't have permissions). * uncomment test * fix * CR * CR
microsoft · Feb 14, 2024 · 4314382 · 4314382
1 parent 4dfe135
commit 4314382
Show file tree

Hide file tree

Showing 12 changed files with 632 additions and 23 deletions.
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
@@ -0,0 +1,47 @@
+name: Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build-and-push:
+    permissions: write-all
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Scantools Docker image
+        run: |
+          DATE=$(date +%Y-%m-%d)
+          docker build . --tag ghcr.io/microsoft/lamar-benchmark/scantools:$DATE --target scantools
+          if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
+            docker tag ghcr.io/microsoft/lamar-benchmark/scantools:$DATE \
+                        ghcr.io/microsoft/lamar-benchmark/scantools:latest
+            docker push ghcr.io/microsoft/lamar-benchmark/scantools:$DATE
+            docker push ghcr.io/microsoft/lamar-benchmark/scantools:latest
+          fi
+
+      - name: Build and push Lamar Docker image
+        run: |
+          DATE=$(date +%Y-%m-%d)
+          docker build . --tag ghcr.io/microsoft/lamar-benchmark/lamar:$DATE --target lamar
+          if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
+            docker tag ghcr.io/microsoft/lamar-benchmark/lamar:$DATE \
+                       ghcr.io/microsoft/lamar-benchmark/lamar:latest
+            docker push ghcr.io/microsoft/lamar-benchmark/lamar:$DATE
+            docker push ghcr.io/microsoft/lamar-benchmark/lamar:latest
+          fi
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,176 @@
+ARG UBUNTU_VERSION=22.04
+FROM mcr.microsoft.com/mirror/docker/library/ubuntu:${UBUNTU_VERSION} AS common
+
+# Minimal toolings.
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends --no-install-suggests \
+        bash \
+        git \
+        python-is-python3 \
+        python3-minimal \
+        python3-pip \
+        sudo \
+        wget
+
+RUN python3 -m pip install --upgrade pip
+
+ADD . /lamar
+
+#
+# Builder stage.
+#
+FROM common AS builder
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends --no-install-suggests \
+        build-essential \
+        cmake \
+        libeigen3-dev \
+        python3-dev \
+        python3-setuptools
+
+# Build raybender.
+COPY docker/scripts/build_raybender.sh /tmp/
+RUN bash /tmp/build_raybender.sh && rm /tmp/build_raybender.sh
+
+# Build pcdmeshing.
+COPY docker/scripts/build_pcdmeshing.sh /tmp/
+RUN bash /tmp/build_pcdmeshing.sh && rm /tmp/build_pcdmeshing.sh
+
+# Build hloc.
+COPY docker/scripts/build_hloc.sh /tmp/
+RUN bash /tmp/build_hloc.sh && rm /tmp/build_hloc.sh
+
+#
+# Scantools stage.
+#
+FROM common AS scantools
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends --no-install-suggests \
+        libgomp1 \
+        libgl1 \
+        libglib2.0-0 \
+        libsm6 \
+        libxrender1 \
+        libxext6 \
+        libzbar0
+
+# Install raybender.
+COPY --from=builder /raybender/embree-3.12.2/lib /raybender/embree-3.12.2/lib
+COPY --from=builder /raybender/dist-wheel /tmp/dist-wheel
+RUN cd /tmp && whl_path=$(cat dist-wheel/whl_path.txt) && python3 -m pip install $whl_path
+RUN rm -rfv /tmp/*
+
+# Install pcdmeshing.
+COPY --from=builder /pcdmeshing/dist-wheel /tmp/dist-wheel
+RUN sudo apt-get install -y --no-install-recommends --no-install-suggests \
+        libmpfrc++-dev
+RUN cd /tmp && whl_path=$(cat dist-wheel/whl_path.txt) && python3 -m pip install $whl_path
+RUN rm -rfv /tmp/*
+
+RUN python3 -m pip install --no-deps \
+        astral==3.2 \
+        beautifulsoup4==4.12.2 \
+        lxml==4.9.2 \
+        matplotlib \
+        open3d==0.18.0 \
+        opencv-python==4.7.0.72 \
+        plyfile==1.0.3 \
+        pytijo==0.0.2 \
+        pyzbar-upright==0.1.8 \
+        scipy==1.11.4
+RUN cd lamar && python3 -m pip install -e .[scantools] --no-deps
+WORKDIR /lamar
+
+#
+# pyceres-builder stage.
+#
+FROM mcr.microsoft.com/mirror/docker/library/ubuntu:${UBUNTU_VERSION} AS pyceres-builder
+
+# Prepare and empty machine for building.
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends --no-install-suggests \
+        git \
+        cmake \
+        ninja-build \
+        build-essential \
+        libeigen3-dev \
+        libgoogle-glog-dev \
+        libgflags-dev \
+        libgtest-dev \
+        libatlas-base-dev \
+        libsuitesparse-dev \
+        python-is-python3 \
+        python3-minimal \
+        python3-pip \
+        python3-dev \
+        python3-setuptools
+
+# Install Ceres.
+RUN apt-get install -y --no-install-recommends --no-install-suggests wget && \
+    wget "http://ceres-solver.org/ceres-solver-2.1.0.tar.gz" && \
+    tar zxf ceres-solver-2.1.0.tar.gz && \
+    mkdir ceres-build && \
+    cd ceres-build && \
+    cmake ../ceres-solver-2.1.0 -GNinja \
+        -DCMAKE_INSTALL_PREFIX=/ceres_installed && \
+    ninja install
+RUN cp -r /ceres_installed/* /usr/local/
+
+# Build pyceres.
+RUN git clone --depth 1 --recursive https://github.com/cvg/pyceres
+RUN python3 -m pip install --upgrade pip
+RUN cd pyceres && \
+    pip wheel . --no-deps -w dist-wheel -vv && \
+    whl_path=$(find dist-wheel/ -name "*.whl") && \
+    echo $whl_path >dist-wheel/whl_path.txt
+
+#
+# pyceres stage.
+#
+FROM scantools as pyceres
+
+# Install minimal runtime dependencies.
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends --no-install-suggests \
+        libgoogle-glog0v5 \
+        libspqr2 \
+        libcxsparse3 \
+        libatlas3-base \
+        python-is-python3 \
+        python3-minimal \
+        python3-pip
+
+# Copy installed library in the builder stage.
+COPY --from=pyceres-builder /ceres_installed/ /usr/local/
+
+# Install pyceres.
+COPY --from=pyceres-builder /pyceres/dist-wheel /tmp/dist-wheel
+RUN pip install --upgrade pip
+RUN cd /tmp && whl_path=$(cat dist-wheel/whl_path.txt) && pip install $whl_path
+RUN rm -rfv /tmp/*
+
+#
+# lamar stage.
+#
+FROM pyceres as lamar
+
+# Install hloc.
+COPY --from=builder /hloc/dist-wheel /tmp/dist-wheel
+RUN cd /tmp && whl_path=$(cat dist-wheel/whl_path.txt) && python3 -m pip install $whl_path
+RUN rm -rfv /tmp/*
+
+# Note: The dependencies listed in pyproject.toml also include pyceres, already
+# installed in previous Docker stages. Attempting to compile it in this stage
+# will lead to failure due to missing necessary development dependencies.
+# Therefore, we replicate the dependencies here, excluding pyceres
+RUN python3 -m pip install --no-deps \
+        h5py==3.10.0 \
+        numpy==1.26.3 \
+        torch>=1.1 \
+        tqdm>=4.36.0 \
+        pycolmap==0.6.0
+
+RUN cd /lamar && python3 -m pip install -e .  --no-deps
+WORKDIR /lamar
diff --git a/README.md b/README.md
@@ -68,8 +68,9 @@ T_w_i = sessions.trajectories[keys[0]]  # first pose, from sensor/rig to world
 
 :one: Install the core dependencies:
 
-- Python >= 3.8
-- [hloc](https://github.com/cvg/Hierarchical-Localization) and its dependencies, including [COLMAP](https://colmap.github.io/install.html) built from source
+- Python >= 3.9
+- [hloc](https://github.com/cvg/Hierarchical-Localization) and its dependencies, including [COLMAP](https://colmap.github.io/install.html) built from source.
+- [pyceres][https://github.com/cvg/pyceres.git] built from source.
 
 :two: Install the LaMAR libraries and pull the remaining pip dependencies:
 ```bash
@@ -87,6 +88,35 @@ python -m pip install -e .
 python -m pip install -e .[dev]
 ```
 
+## Docker images
+
+The Dockerfile provided in this project has multiple stages, two of which are:
+`scantools` and `lamar`.
+
+### Building the Docker Images
+
+You can build the Docker images for these stages using the following commands:
+```bash
+# Build the 'scantools' stage
+docker build --target scantools -t lamar:scantools -f Dockerfile ./
+
+# Build the 'lamar' stage
+docker build --target lamar -t lamar:lamar -f Dockerfile ./
+```
+
+### Pulling the Docker Images from GitHub Docker Registry
+
+Alternatively, if you don't want to build the images yourself, you can pull them
+from the GitHub Docker Registry using the following commands:
+```bash
+# Pull the 'scantools' image
+docker pull ghcr.io/microsoft/lamar-benchmark/scantools:latest
+
+# Pull the 'lamar' image
+docker pull ghcr.io/microsoft/lamar-benchmark/lamar:latest
+```
+
+
 ## Benchmark
 
 :one: __Obtain the evaluation data:__ [visit the dataset page](https://lamar.ethz.ch/lamar/) and place the 3 scenes in `./data` :

diff --git a/docker/scripts/build_hloc.sh b/docker/scripts/build_hloc.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+PS4='\033[1;96m$(date +%H:%M:%S)\033[0m '
+set -exo pipefail
+
+# Clone hloc.
+git clone --recursive https://github.com/cvg/Hierarchical-Localization/ hloc --depth=1
+cd hloc
+
+# Build the wheel.
+pip wheel --no-deps -w dist-wheel .
+whl_path=$(find dist-wheel/ -name "*.whl")
+echo $whl_path >dist-wheel/whl_path.txt
diff --git a/docker/scripts/build_pcdmeshing.sh b/docker/scripts/build_pcdmeshing.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+PS4='\033[1;96m$(date +%H:%M:%S)\033[0m '
+set -exo pipefail
+
+sudo apt-get install -y --no-install-recommends --no-install-suggests \
+         libboost-dev libgmp3-dev libmpfrc++-dev
+git clone --recursive https://github.com/cvg/pcdmeshing.git --depth=1
+cd pcdmeshing
+
+# Build the wheel.
+pip wheel --no-deps -w dist-wheel .
+whl_path=$(find dist-wheel/ -name "*.whl")
+echo $whl_path >dist-wheel/whl_path.txt
diff --git a/docker/scripts/build_raybender.sh b/docker/scripts/build_raybender.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+PS4='\033[1;96m$(date +%H:%M:%S)\033[0m '
+set -exo pipefail
+
+# Clone raybender.
+git clone --recursive https://github.com/cvg/raybender.git --depth=1
+cd raybender
+
+# Install Embree following the official instructions and set the environmental
+# variable embree_DIR to point to embree-config.cmake. On Linux, this can be
+# done as follows:
+wget https://github.com/embree/embree/releases/download/v3.12.2/embree-3.12.2.x86_64.linux.tar.gz
+tar xvzf embree-3.12.2.x86_64.linux.tar.gz
+rm embree-3.12.2.x86_64.linux.tar.gz
+mv embree-3.12.2.x86_64.linux embree-3.12.2
+export embree_DIR=`readlink -f embree-3.12.2/lib/cmake/embree-3.12.2`
+
+# Build the wheel.
+pip wheel --no-deps -w dist-wheel .
+whl_path=$(find dist-wheel/ -name "*.whl")
+echo $whl_path >dist-wheel/whl_path.txt