CI: packaging tests (#261)

.github/workflows/tests.yml

@@ -52,18 +52,37 @@ jobs:
     - name: Install functional test dependencies
       run: |
         sudo apt install -y nftables iptables ipset
+        docker network create net-test
+        python3 -m pip install --upgrade pipenv wheel
+        pipenv install --deploy
+        # some tests need root, so we have to install pytest twice
         sudo python3 -m pip install --upgrade pipenv wheel
         sudo pipenv install --deploy
-        docker network create net-test
 
     - name: Run functional tests
       env:
         CROWDSEC_TEST_VERSION: dev
         CROWDSEC_TEST_FLAVORS: full
         CROWDSEC_TEST_NETWORK: net-test
         CROWDSEC_TEST_TIMEOUT: 60
+        PYTEST_ADDOPTS: --durations=0 -vv --color=yes -m "not (deb or rpm)"
       run: |
-        sudo pipenv run pytest --durations=0 --color=yes
+        # everything except for
+        # - install (requires root, ignored by default)
+        # - backends (requires root, ignored by default)
+        # - deb/rpm (on their own workflows)
+        pipenv run pytest
+        # these need root
+        sudo -E pipenv run pytest ./test/backends
+        sudo -E pipenv run pytest ./test/install/no_crowdsec
+        # these need a running crowdsec
+        docker run -d --name crowdsec -e CI_TESTING=true -e DISABLE_ONLINE_API=true -ti crowdsecurity/crowdsec
+        install -m 0755 /dev/stdin /usr/local/bin/cscli <<'EOT'
+        #!/bin/sh
+        docker exec crowdsec cscli "$@"
+        EOT
+        sleep 5
+        sudo -E pipenv run pytest ./test/install/with_crowdsec
 
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v3

.github/workflows/tests_deb.yml

@@ -0,0 +1,55 @@
+name: Test .deb packaging
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: "Test .deb packages"
+    runs-on: ubuntu-latest
+
+    steps:
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: 1.20.3
+
+    - name: Cache virtualenvs
+      id: cache-pipenv
+      uses: actions/cache@v3
+      with:
+        path: ~/.local/share/virtualenvs
+        key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
+
+    - name: Install functional test dependencies
+      run: |
+        sudo apt install -y build-essential debhelper devscripts fakeroot lintian
+        docker network create net-test
+        python3 -m pip install --upgrade pipenv wheel
+        pipenv install --deploy
+        sudo python3 -m pip install --upgrade pipenv wheel
+        sudo pipenv install --deploy
+
+    - name: Run functional tests
+      env:
+        CROWDSEC_TEST_VERSION: dev
+        CROWDSEC_TEST_FLAVORS: full
+        CROWDSEC_TEST_NETWORK: net-test
+        CROWDSEC_TEST_TIMEOUT: 60
+        PYTEST_ADDOPTS: --durations=0 -vv --color=yes
+      run: |
+        sudo apt install -y nftables iptables ipset
+        pipenv run pytest test/pkg/test_build_deb.py
+        sudo -E pipenv run pytest -m deb ./test/install/no_crowdsec

.gitignore

@@ -25,3 +25,10 @@ venv/
 /debian/*.substvars
 /debian/*.debhelper
 /debian/*-stamp
+
+# built by rpmbuild
+/rpm/BUILD
+/rpm/BUILDROOT
+/rpm/RPMS
+/rpm/SOURCES/*.tar.gz
+/rpm/SRPMS

Makefile

@@ -20,12 +20,15 @@ LD_OPTS_VARS=\
 -X '$(GO_MODULE_NAME)/pkg/version.BuildDate=$(BUILD_TIMESTAMP)' \
 -X '$(GO_MODULE_NAME)/pkg/version.Tag=$(BUILD_TAG)'
 
+export CGO_ENABLED=0
 export LD_OPTS=-ldflags "-a -s -w -extldflags '-static' $(LD_OPTS_VARS)" \
 	-trimpath -tags netgo
 
 .PHONY: all
 all: build test
 
+# same as "$(MAKE) -f debian/rules clean" but without the dependency on debhelper
+.PHONY: clean-debian
 clean-debian:
 	@$(RM) -r debian/crowdsec-firewall-bouncer-iptables
 	@$(RM) -r debian/crowdsec-firewall-bouncer-nftables
@@ -34,9 +37,17 @@ clean-debian:
 	@$(RM) -r debian/*.substvars
 	@$(RM) -r debian/*-stamp
 
+.PHONY: clean-rpm
+clean-rpm:
+	@$(RM) -r rpm/BUILD
+	@$(RM) -r rpm/BUILDROOT
+	@$(RM) -r rpm/RPMS
+	@$(RM) -r rpm/SOURCES/*.tar.gz
+	@$(RM) -r rpm/SRPMS
+
 # Remove everything including all platform binaries and tarballs
 .PHONY: clean
-clean: clean-release-dir clean-debian
+clean: clean-release-dir clean-debian clean-rpm
 	@$(RM) $(BINARY_NAME)
 	@$(RM) $(TARBALL_NAME)
 	@$(RM) -r $(BINARY_NAME)-*	# platform binary name and leftover release dir

Pipfile

@@ -1,13 +1,15 @@
 [packages]
-pytest-dotenv = "*"
-flask = "*"
-pytimeparse = "*"
-psutil = "*"
-pytest-cs = {ref = "0.6.0", git = "https://github.com/crowdsecurity/pytest-cs.git"}
+pytest-cs = {ref = "0.7.13", git = "https://github.com/crowdsecurity/pytest-cs.git"}
+pytest-dotenv = "0.5.2"
+pytest-dependency = "0.5.1"
+pexpect = "4.8.0"
+flask = "2.2.3"
+pytimeparse = "1.1.8"
+psutil = "5.9.5"
 
 [dev-packages]
-gnureadline = "*"
-ipdb = "*"
+gnureadline = "8.1.2"
+ipdb = "0.13.13"
 
 [requires]
 python_version = "3.10"

debian/crowdsec-firewall-bouncer-iptables.postinst

@@ -2,9 +2,6 @@
 
 systemctl daemon-reload
 
-BOUNCER="crowdsec-firewall-bouncer"
-BOUNCER_PREFIX="FirewallBouncer"
-
 #shellcheck source=./scripts/_bouncer.sh
 . "/usr/lib/$DPKG_MAINTSCRIPT_PACKAGE/_bouncer.sh"
 START=1

debian/crowdsec-firewall-bouncer-nftables.postinst

@@ -2,9 +2,6 @@
 
 systemctl daemon-reload
 
-BOUNCER="crowdsec-firewall-bouncer"
-BOUNCER_PREFIX="FirewallBouncer"
-
 #shellcheck source=./scripts/_bouncer.sh
 . "/usr/lib/$DPKG_MAINTSCRIPT_PACKAGE/_bouncer.sh"
 START=1