Merge branch 'master' into feat/acquisition/add-http-datasource

crowdsecurity · Oct 23, 2024 · 2cc5c4e · 2cc5c4e
2 parents 5577e10 + d8bc17b
commit 2cc5c4e
Show file tree

Hide file tree

Showing 836 changed files with 63,204 additions and 35,380 deletions.
diff --git a/.github/codecov.yml b/.github/codecov.yml
diff --git a/.github/generate-codecov-yml.sh b/.github/generate-codecov-yml.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Run this from the repository root:
+#
+# .github/generate-codecov-yml.sh >> .github/codecov.yml
+
+cat <<EOT
+# we measure coverage but don't enforce it
+# https://docs.codecov.com/docs/codecov-yaml
+codecov:
+  require_ci_to_pass: false
+
+coverage:
+  status:
+    patch:
+      default:
+        target: 0%
+    project:
+      default:
+        target: 0%
+
+# if a directory is ignored, there is no way to un-ignore files like pkg/models/helpers.go
+# so we make a full list
+ignore:
+EOT
+
+find . -name "*.go" | while read -r file; do
+    if head -n 1 "$file" | grep -q "Code generated by"; then
+        echo "  - \"$file\""
+    fi
+done
diff --git a/.github/governance.yml b/.github/governance.yml
@@ -42,7 +42,7 @@ issue:
           3. Check [Releases](https://github.com/crowdsecurity/crowdsec/releases/latest) to make sure your agent is on the latest version.
 
     - prefix: kind
-      list: ['feature', 'bug', 'packaging', 'enhancement']
+      list: ['feature', 'bug', 'packaging', 'enhancement', 'refactoring']
       multiple: false
       author_association:
         author: true
@@ -54,6 +54,7 @@ issue:
           @$AUTHOR: There are no 'kind' label on this issue. You need a 'kind' label to start the triage process.
           * `/kind feature`
           * `/kind enhancement`
+          * `/kind refactoring`
           * `/kind bug`
           * `/kind packaging`
 
@@ -65,12 +66,13 @@ pull_request:
   labels:
     - prefix: kind
       multiple: false
-      list: [ 'feature', 'enhancement', 'fix', 'chore', 'dependencies']
+      list: [ 'feature', 'enhancement', 'fix', 'chore', 'dependencies', 'refactoring']
       needs:
         comment: |
           @$AUTHOR: There are no 'kind' label on this PR. You need a 'kind' label to generate the release automatically.
           * `/kind feature`
           * `/kind enhancement`
+          * `/kind refactoring`
           * `/kind fix`
           * `/kind chore`
           * `/kind dependencies`
@@ -81,14 +83,15 @@ pull_request:
             failure: Missing kind label to generate release automatically.
 
     - prefix: area
-      list: [ "agent", "local-api", "cscli", "security", "configuration"]
+      list: [ "agent", "local-api", "cscli", "security", "configuration", "appsec"]
       multiple: true
       needs:
         comment: |
           @$AUTHOR: There are no area labels on this PR. You can add as many areas as you see fit.
           * `/area agent`
           * `/area local-api`
           * `/area cscli`
+          * `/area appsec`
           * `/area security`
           * `/area configuration`
 
@@ -98,4 +101,4 @@ pull_request:
       author_association:
         collaborator: true
         member: true
-        owner: true
+        owner: true
diff --git a/.github/workflows/bats-hub.yml b/.github/workflows/bats-hub.yml
@@ -1,4 +1,4 @@
-name: Hub tests
+name: (sub) Bats / Hub
 
 on:
   workflow_call:
@@ -8,16 +8,13 @@ on:
       GIST_BADGES_ID:
         required: true
 
-env:
-  PREFIX_TEST_NAMES_WITH_FILE: true
-
 jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        test-file: ["hub-1.bats", "hub-2.bats", "hub-3.bats"]
 
-    name: "Build + tests"
+    name: "Functional tests"
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
@@ -28,34 +25,35 @@ jobs:
           echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
 
     - name: "Check out CrowdSec repository"
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         submodules: true
 
-    - name: "Set up Go ${{ matrix.go-version }}"
-      uses: actions/setup-go@v4
+    - name: "Set up Go"
+      uses: actions/setup-go@v5
       with:
-        go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
+        go-version: "1.22"
 
     - name: "Install bats dependencies"
       env:
         GOBIN: /usr/local/bin
       run: |
-        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
+        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
 
     - name: "Build crowdsec and fixture"
       run: make bats-clean bats-build bats-fixture BUILD_STATIC=1
 
     - name: "Run hub tests"
-      run: make bats-test-hub
+      run: |
+          ./test/bin/generate-hub-tests
+          ./test/run-tests ./test/dyn-bats/${{ matrix.test-file }} --formatter $(pwd)/test/lib/color-formatter
 
     - name: "Collect hub coverage"
       run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV
 
     - name: "Create Parsers badge"
-      uses: schneegans/dynamic-badges-action@v1.6.0
+      uses: schneegans/dynamic-badges-action@v1.7.0
       if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'crowdsecurity' }}
       with:
         auth: ${{ secrets.GIST_BADGES_SECRET }}
@@ -66,7 +64,7 @@ jobs:
         color: ${{ env.SCENARIO_BADGE_COLOR }}
 
     - name: "Create Scenarios badge"
-      uses: schneegans/dynamic-badges-action@v1.6.0
+      uses: schneegans/dynamic-badges-action@v1.7.0
       if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'crowdsecurity' }}
       with:
         auth: ${{ secrets.GIST_BADGES_SECRET }}

diff --git a/.github/workflows/bats-mysql.yml b/.github/workflows/bats-mysql.yml
@@ -1,4 +1,4 @@
-name: Functional tests (MySQL)
+name: (sub) Bats / MySQL
 
 on:
   workflow_call:
@@ -7,16 +7,9 @@ on:
         required: true
         type: string
 
-env:
-  PREFIX_TEST_NAMES_WITH_FILE: true
-
 jobs:
   build:
-    strategy:
-      matrix:
-        go-version: ["1.20.6"]
-
-    name: "Build + tests"
+    name: "Functional tests"
     runs-on: ubuntu-latest
     timeout-minutes: 30
     services:
@@ -35,22 +28,21 @@ jobs:
           echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
 
     - name: "Check out CrowdSec repository"
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         submodules: true
 
-    - name: "Set up Go ${{ matrix.go-version }}"
-      uses: actions/setup-go@v4
+    - name: "Set up Go"
+      uses: actions/setup-go@v5
       with:
-        go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
+        go-version: "1.22"
 
     - name: "Install bats dependencies"
       env:
         GOBIN: /usr/local/bin
       run: |
-        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
+        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
 
     - name: "Build crowdsec and fixture"
       run: |
@@ -63,7 +55,7 @@ jobs:
         MYSQL_USER: root
 
     - name: "Run tests"
-      run: make bats-test
+      run: ./test/run-tests ./test/bats --formatter $(pwd)/test/lib/color-formatter
       env:
         DB_BACKEND: mysql
         MYSQL_HOST: 127.0.0.1

diff --git a/.github/workflows/bats-postgres.yml b/.github/workflows/bats-postgres.yml
@@ -1,23 +1,16 @@
-name: Functional tests (Postgres)
+name: (sub) Bats / Postgres
 
 on:
   workflow_call:
 
-env:
-  PREFIX_TEST_NAMES_WITH_FILE: true
-
 jobs:
   build:
-    strategy:
-      matrix:
-        go-version: ["1.20.6"]
-
-    name: "Build + tests"
+    name: "Functional tests"
     runs-on: ubuntu-latest
     timeout-minutes: 30
     services:
       database:
-        image: postgres:15
+        image: postgres:16
         env:
           POSTGRES_PASSWORD: "secret"
         ports:
@@ -30,36 +23,35 @@ jobs:
 
     steps:
 
-    - name: "Install pg_dump v15"
+    - name: "Install pg_dump v16"
       # we can remove this when it's released on ubuntu-latest
       run: |
           sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
           wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc &>/dev/null
           sudo apt update
-          sudo apt -qq -y -o=Dpkg::Use-Pty=0 install postgresql-client-15
+          sudo apt -qq -y -o=Dpkg::Use-Pty=0 install postgresql-client-16
 
     - name: "Force machineid"
       run: |
           sudo chmod +w /etc/machine-id
           echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
 
     - name: "Check out CrowdSec repository"
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         submodules: true
 
-    - name: "Set up Go ${{ matrix.go-version }}"
-      uses: actions/setup-go@v4
+    - name: "Set up Go"
+      uses: actions/setup-go@v5
       with:
-        go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
+        go-version: "1.22"
 
     - name: "Install bats dependencies"
       env:
         GOBIN: /usr/local/bin
       run: |
-        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
+        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
 
     - name: "Build crowdsec and fixture (DB_BACKEND: pgx)"
       run: |
@@ -72,7 +64,7 @@ jobs:
         PGUSER: postgres
 
     - name: "Run tests (DB_BACKEND: pgx)"
-      run: make bats-test
+      run: ./test/run-tests ./test/bats --formatter $(pwd)/test/lib/color-formatter
       env:
         DB_BACKEND: pgx
         PGHOST: 127.0.0.1

diff --git a/.github/workflows/bats-sqlite-coverage.yml b/.github/workflows/bats-sqlite-coverage.yml
@@ -1,19 +1,17 @@
-name: Functional tests (sqlite)
+name: (sub) Bats / sqlite + coverage
 
 on:
   workflow_call:
+    secrets:
+      CODECOV_TOKEN:
+        required: true
 
 env:
-  PREFIX_TEST_NAMES_WITH_FILE: true
   TEST_COVERAGE: true
 
 jobs:
   build:
-    strategy:
-      matrix:
-        go-version: ["1.20.6"]
-
-    name: "Build + tests"
+    name: "Functional tests"
     runs-on: ubuntu-latest
     timeout-minutes: 20
 
@@ -25,29 +23,32 @@ jobs:
           echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
 
     - name: "Check out CrowdSec repository"
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         submodules: true
 
-    - name: "Set up Go ${{ matrix.go-version }}"
-      uses: actions/setup-go@v4
+    - name: "Set up Go"
+      uses: actions/setup-go@v5
       with:
-        go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
+        go-version: "1.22"
 
     - name: "Install bats dependencies"
       env:
         GOBIN: /usr/local/bin
       run: |
-        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
+        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
 
     - name: "Build crowdsec and fixture"
       run: |
         make clean bats-build bats-fixture BUILD_STATIC=1
 
+    - name: Generate codecov configuration
+      run: |
+          .github/generate-codecov-yml.sh >> .github/codecov.yml
+
     - name: "Run tests"
-      run: make bats-test
+      run: ./test/run-tests ./test/bats --formatter $(pwd)/test/lib/color-formatter
 
     - name: "Collect coverage data"
       run: |
@@ -82,8 +83,9 @@ jobs:
       run: for file in $(find ./test/local/var/log -type f); do echo ">>>>> $file"; cat $file; echo; done
       if: ${{ always() }}
 
-    - name: Upload crowdsec coverage to codecov
-      uses: codecov/codecov-action@v3
+    - name: Upload bats coverage to codecov
+      uses: codecov/codecov-action@v4
       with:
         files: ./coverage-bats.out
         flags: bats
+        token: ${{ secrets.CODECOV_TOKEN }}