Update to python 3.11, improve build times (MISP#26)

Changes: * misp-modules targets py3.11, so use that base, + ninja + wheels groundwork * slightly optimise local builder layer cache size * pin libfaup to specific commit * remove comment * move libfaup_commit to template, get apios from pypi and let it decide validators version
dgujarathi · Sep 12, 2023 · c0ea24e · c0ea24e
1 parent 4c890ee
commit c0ea24e
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 21 deletions.
diff --git a/docker-bake.hcl b/docker-bake.hcl
@@ -18,6 +18,10 @@ variable "MODULES_COMMIT" {
   default = ""
 }
 
+variable "LIBFAUP_COMMIT" {
+  default = null
+}
+
 variable "MISP_TAG" {
   default = ""
 }

diff --git a/modules/Dockerfile b/modules/Dockerfile
@@ -1,9 +1,10 @@
 ARG DOCKER_HUB_PROXY=""
 
-FROM "${DOCKER_HUB_PROXY}python:3.9-slim-bullseye" as python-build
+FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build
     ENV DEBIAN_FRONTEND noninteractive
     ARG MODULES_TAG
     ARG MODULES_COMMIT
+    ARG LIBFAUP_COMMIT
     RUN apt-get update && apt-get install -y --no-install-recommends \
         cmake \
         git \
@@ -15,33 +16,45 @@ FROM "${DOCKER_HUB_PROXY}python:3.9-slim-bullseye" as python-build
         libpoppler-cpp-dev \
         libfuzzy-dev \
         libssl-dev \
+        ninja-build \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
     RUN mkdir /wheels
     WORKDIR /srv
 
-    RUN if [ ! -z ${MODULES_COMMIT} ]; then \
-        git clone https://github.com/MISP/misp-modules.git /srv/misp-modules && cd /srv/misp-modules && git checkout ${MODULES_COMMIT}; \
-        else git clone --branch ${MODULES_TAG} --depth 1 https://github.com/MISP/misp-modules.git /srv/misp-modules; fi
+    RUN <<-EOF
+            if [ ! -z ${MODULES_COMMIT} ]; then
+                git clone https://github.com/MISP/misp-modules.git /srv/misp-modules && cd /srv/misp-modules && git checkout ${MODULES_COMMIT}
+            else
+                git clone --branch ${MODULES_TAG} --depth 1 https://github.com/MISP/misp-modules.git /srv/misp-modules
+            fi
 
-    RUN cd /srv/misp-modules; \
-        echo "pyeti" >> REQUIREMENTS; \
-        echo "greynoise" >> REQUIREMENTS; \
-        echo "git+https://github.com/abenassi/Google-Search-API" >> REQUIREMENTS; \
-        sed -i 's|apiosintds.*|git+https://github.com/davidonzo/apiosintDS\@misp|g' REQUIREMENTS; \
-        sed -i 's/-e //g' REQUIREMENTS; \
-        pip3 wheel -r REQUIREMENTS --no-cache-dir -w /wheels/
+            cd /srv/misp-modules
+            echo "pyeti" >> REQUIREMENTS
+            echo "greynoise" >> REQUIREMENTS
+            echo "git+https://github.com/abenassi/Google-Search-API" >> REQUIREMENTS
+            sed -i 's|.*apiosintDS.*|apiosintDS==2.0.1|g' REQUIREMENTS
+            # let apiosintDS decide since misp-modules doesn't actually use this package anyway
+            sed -i '/validators.*/d' REQUIREMENTS
+            sed -i 's/-e //g' REQUIREMENTS
+            pip3 wheel -r REQUIREMENTS --no-cache-dir -w /wheels/
+            rm -rf /srv/misp-modules
+EOF
 
-    RUN git clone --depth 1 https://github.com/stricaud/faup.git /srv/faup; \
-        cd /srv/faup/build || exit; cmake .. && make install; \
-        cd /srv/faup/src/lib/bindings/python || exit; pip3 wheel --no-cache-dir -w /wheels/ .
+    RUN <<-EOF
+        git clone --depth 1 https://github.com/stricaud/faup.git /srv/faup
+        cd /srv/faup
+        git checkout ${LIBFAUP_COMMIT}
+        cd /srv/faup/build
+        cmake -G "Ninja" ../
+        ninja
+        ninja install
+        cd /srv/faup/src/lib/bindings/python
+        pip3 wheel --no-cache-dir --no-dependencies -w /wheels/ .
+        rm -rf /srv/faup
+EOF
 
-    # Remove extra packages due to incompatible requirements.txt files
-    WORKDIR /wheels
-    RUN find . -name "chardet*" | grep -v "chardet-4.0.0" | xargs rm -f
-
-
-FROM "${DOCKER_HUB_PROXY}python:3.9-slim-bullseye"
+FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye"
     ENV DEBIAN_FRONTEND noninteractive
     RUN apt-get update && apt-get install -y --no-install-recommends \
         libglib2.0-0 \
@@ -59,7 +72,7 @@ FROM "${DOCKER_HUB_PROXY}python:3.9-slim-bullseye"
     # Since we compile faup ourselves and lua is not required anymore, we can load our own library 
     #   and skip the pre-compiled blob to improve compatibility with other architectures like ARM
     RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \
-        /usr/local/lib/python3.9/site-packages/pyfaup/__init__.py
+        /usr/local/lib/python3.11/site-packages/pyfaup/__init__.py
 
     # Disable (all) warnings raised when using 'future'
     RUN sed -i '/import sys/a import warnings\nwarnings.warn = lambda *args, **kwargs: None' \

diff --git a/template.env b/template.env
@@ -1,6 +1,7 @@
 MISP_TAG=v2.4.175
 MODULES_TAG=v2.4.175
 PHP_VER=20190902
+LIBFAUP_COMMIT=3a26d0a
 
 # MISP_COMMIT takes precedence over MISP_TAG
 # MISP_COMMIT=c56d537