Merge pull request #4 from cmu-sei/test_container

Creating an integration test container

.github/workflows/ci.yaml

@@ -26,13 +26,7 @@ jobs:
         run: make check-lint
       - name: Check types
         run: make check-typecheck  
-      - name: Execute unit tests
-        run: make test
-
-  docker:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v4
-      - name: Docker Build Action
-        run: docker build -t vessel .
+      - name: Build main and test containers
+        run: make build-containers
+      - name: Execute unit tests inside test container
+        run: make test-container

Dockerfile

@@ -1,5 +1,7 @@
 FROM python:3.11-bookworm
 
+# General dependencies, as well as diffoscope-specific sub-dependencies for
+# its specific diff plugins.
 RUN apt-get update && \
   apt-get install -y \
     skopeo=1.9.3+ds1-1+b9 \
@@ -54,33 +56,44 @@ RUN apt-get update && \
     xxd=2:9.0.1378-2 \
     xmlbeans=4.0.0-2 \
     xxd=2:9.0.1378-2 \
-    python3-guestfs=1:1.48.6-2 
+    python3-guestfs=1:1.48.6-2  \
+    ca-certificates
 
+# Set up certificates for any proxies that can get in the middle of curl/wget commands during the build
+# NOTE: put any CA certificates needed for a proxy in the ./certs folder in the root of this repo, in PEM format
+# but with a .crt extension, so they can be loaded into the container and used for SSL connections properly.
+RUN mkdir /certs
+COPY ./certs/ /certs/
+RUN if [ -n "$(ls -A /certs/*.crt)" ]; then \
+      cp -rf /certs/*.crt /usr/local/share/ca-certificates/; \
+      update-ca-certificates; \
+    fi
+
+# Get another sub-dependency for diffoscope.
 RUN git clone https://github.com/radareorg/radare2.git \
   && cd radare2 \
   && ./sys/install.sh \
   && rm -rf /radare2
 
+# Set up workdir and env vars.
 ENV WORKDIR=/opt/project
 WORKDIR ${WORKDIR}
-
 ENV VENV_PATH="${WORKDIR}/.venv"
 ENV PATH="${VENV_PATH}/bin:$PATH"
 
 # Install poetry and set up venv. 
 RUN python -m venv ${VENV_PATH} \
-  && python -m pip install poetry==1.8.2
-
-COPY ./pyproject.toml ./poetry.lock ./README.md ${WORKDIR}
+  && python -m pip install poetry==2.0.1
 
 # Install Python dependencies.
+COPY ./pyproject.toml ./poetry.lock ./README.md ${WORKDIR}
 RUN poetry install -vv --no-cache --no-root --no-interaction --with extra_dependencies \
     && rm -rf /root/.cache/pypoetry/*
 
 # Copy our app.
 COPY ./vessel ${WORKDIR}/vessel  
 
-# Install Python dependencies.
+# Install Vessel itself.
 RUN poetry install -vv --no-cache --only-root --no-interaction \
     && rm -rf /root/.cache/pypoetry/*
 

Dockerfile.test

@@ -0,0 +1,13 @@
+FROM vessel
+
+# Install test deps.
+RUN poetry install -vv --no-cache --no-root --no-interaction --with qa \
+    && rm -rf /root/.cache/pypoetry/*
+
+# Copy tests.
+ENV WORKDIR=/opt/project
+WORKDIR ${WORKDIR}
+COPY ./test ${WORKDIR}/test  
+
+# Change the entry point so all tests are run instead of vessel.
+ENTRYPOINT ["poetry", "run", "pytest", "test"]

Makefile

@@ -54,13 +54,26 @@ qa: isort format lint typecheck
 .PHONY: check
 check: check-isort check-format check-lint check-typecheck
 
-# Run unit tests with pytest
-.PHONY: test
-test:
-	poetry run pytest test
+# -----------------------------------------------------------------------------
+# Container actions.
+# -----------------------------------------------------------------------------
+
+# Build all containers.
+.PHONY: build-containers
+build-containers:
+	bash build_containers.sh
+
+# Run unit tests inside container
+.PHONY: test-container
+test-container:
+	docker run --rm vessel-test
+
+# Build and run unit tests inside container
+.PHONY: build-test-container
+build-test-container: build-containers test-container
 
 # -----------------------------------------------------------------------------
 # All actions and checks equivalent to what the CI does.
 # -----------------------------------------------------------------------------
 .PHONY: ci
-ci: clean check test
+ci: clean check build-test-container

README.md

@@ -13,7 +13,7 @@ Pre-requisites:
     * Add to `~/.bashrc` (or equivalent profile): `export PATH=~/.local/bin:$PATH`
 
 To set up the Python environment and the required packages:
-1. `poetry shell`
+1. `python -m venv .venv`
 2. `poetry install --with extra_dependencies`
 
 To set up additional external tools that are used:
@@ -39,15 +39,15 @@ Assuming you have Docker installed, run the following to build the vessel docker
 ### In Local Environment
 The tool can be run locally like this:
 
-1. Make sure the environment is active: `poetry shell`
+1. Make sure the environment is active: `source .venv/bin/activate`
 2. Run `sudo env "PATH=$PATH" vessel diff`, with the proper arguments
    * This way of calling it avoids permission issues
 
 Run `vessel --help` for full list of commands and options.
 
 ### Running the Docker container
 
-* Note: Running within Docker avoids permission issues during the unpacking of the images
+* Note: Running within Docker avoids permission issues during the unpacking of the images.
 
 To see commands and options:
 * `docker run --rm vessel --help`

build_containers.sh

@@ -0,0 +1,2 @@
+docker build -t vessel .
+docker build -t vessel-test -f ./Dockerfile.test .

certs/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this folder but itself and the readme.
+*
+!.gitignore
+!readme.txt

certs/readme.txt

@@ -0,0 +1,2 @@
+This folder is only useful if local certs need to be passed to the Docker container to work behind proxies or similar setups.
+See Doeckerfile for details, but it is enough to just put the proper crt files inside this folder for them to be loaded.