tests: add a seeded test to workflow

.dockerignore

@@ -1,5 +1,5 @@
 Dockerfile
 docker-compose.yml
-docker-compose-dev.yml
+docker-compose.yml
 config.json
 paks/

.github/workflows/build.yml

@@ -80,7 +80,7 @@ jobs:
         run: go generate -tags tools -x ./...
 
       - name: Start stack
-        run: docker compose -f docker-compose-dev.yml up -d
+        run: docker compose up -d
 
       - name: Setup Minio
         run: |
@@ -101,3 +101,69 @@ jobs:
         uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
+
+  seeded_test:
+    name: Seeded Test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - download_url: "${{ secrets.S3_URL_PROD }}"
+            s3_bucket: "smr-prod-s3",
+            s3_key: "${{ secrets.S3_KEY_PROD }}",
+            s3_secret: "${{ secrets.S3_SECRET_PROD }}",
+            s3_base_url: "https://storage.ficsit.app"
+          - download_url: "${{ secrets.S3_URL_STAGING }}"
+            s3_bucket: "smr-staging-s3",
+            s3_key: "${{ secrets.S3_KEY_STAGING }}",
+            s3_secret: "${{ secrets.S3_SECRET_STAGING }}",
+            s3_base_url: "https://storage-staging.ficsit.app"
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Download dependencies
+        run: |
+          sudo apt update && sudo apt install -y build-essential libpng-dev protobuf-compiler
+          go install google.golang.org/protobuf/cmd/[email protected]
+          go install google.golang.org/grpc/cmd/[email protected]
+
+      - name: Go Generate
+        run: go generate -tags tools -x ./...
+
+      - name: Start stack
+        run: docker compose up -d
+
+      - name: Setup Minio
+        run: |
+          go install github.com/minio/mc@latest
+          mc alias set local http://localhost:9000 minio minio123
+          mc admin user svcacct add local minio --access-key REPLACE_ME_KEY --secret-key REPLACE_ME_SECRET
+          mc anonymous set public local/smr
+
+      - name: Download and seed database
+        run: |
+          psql -h localhost -p 5432 -U postgres < curl ${{ matrix.download_url }}
+        env:
+          PGPASSWORD: REPLACE_ME
+
+      - name: Test
+        run: go test -v -run '^TestSetup$' ./...
+        env:
+          CGO_ENABLED: 1
+          REPO_PASETO.PUBLIC_KEY: 408c5155a389aeabf1c1b0da73ff5a3079b6aa6628e4c661b1e1ce412181cc8a
+          REPO_PASETO.PRIVATE_KEY: a5f7409588f6b72d443db0d432f37f1214a5ec88cb55a70e24b90194ed549465408c5155a389aeabf1c1b0da73ff5a3079b6aa6628e4c661b1e1ce412181cc8a
+          REPO_FEATURE_FLAGS.ALLOW_MULTI_TARGET_UPLOAD: "true"
+          REPO_STORAGE.READER.TYPE: "s3",
+          REPO_STORAGE.READER.REGION: "us-west-002",
+          REPO_STORAGE.READER.BUCKET: "${{ matrix.s3_bucket }}",
+          REPO_STORAGE.READER.KEY: "${{ matrix.s3_key }}",
+          REPO_STORAGE.READER.SECRET: "${{ matrix.s3_secret }}",
+          REPO_STORAGE.READER.ENDPOINT: "https://s3.us-west-002.backblazeb2.com",
+          REPO_STORAGE.READER.BASE_URL: "${{ matrix.s3_base_url }}",
+          REPO_STORAGE.READER.KEYPATH: "%s/file/%s/%s"

README.md

@@ -30,7 +30,7 @@ To run the API, you will need to have a working Postgres, Redis and Storage. The
 start via:
 
 ```bash
-docker-compose -f docker-compose-dev.yml up -d
+docker compose up -d
 ```
 
 It is suggested you create a configuration file at `config.json` (but you can also use environment variables).

db/statistics.go

@@ -113,6 +113,11 @@ func RunAsyncStatisticLoop(ctx context.Context) {
 			for _, row := range resultRows {
 				err := Tx(ctx, func(ctx context.Context, tx *ent.Tx) error {
 					mod, err := tx.Mod.Get(ctx, row.ModID)
+
+					if mod == nil || mod.ID == "" {
+						return nil
+					}
+
 					if err != nil {
 						return err
 					}

docker-compose.yml

@@ -1,39 +1,62 @@
-version: '2'
-
 services:
-  api:
-    build: .
-    restart: always
-    env_file: .env
-    ports:
-      - 53266:5020
-    networks:
-      internal:
-        ipv4_address: 172.16.239.10
-
   redis:
-    image: redis:alpine
-    restart: always
-    networks:
-      internal:
-        ipv4_address: 172.16.239.2
+    image: redis:7.0.14-alpine
+    ports:
+      - 6379:6379
 
   postgres:
     image: postgres:alpine
     command: postgres -N 500 -B 4096MB
+    environment:
+      POSTGRES_PASSWORD: REPLACE_ME
+    ports:
+      - 5432:5432
+
+  minio:
+    image: quay.io/minio/minio
+    ports:
+      - 9000:9000
+      - 9001:9001
+    entrypoint: sh
+    command: -c 'mkdir -p /data/smr && /usr/bin/docker-entrypoint.sh minio server /data --console-address ":9001"'
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: minio123
+      MINIO_ACCESS_KEY: REPLACE_ME_KEY
+      MINIO_SECRET_KEY: REPLACE_ME_SECRET
+
+  pak_parser:
+    image: ghcr.io/vilsol/ficsit-pak-parser:v0.0.6
+    ports:
+      - 50051:50051
     restart: always
-    env_file: .env
+
+  loki:
+    image: grafana/loki:latest
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+
+  grafana:
+    image: grafana/grafana:latest
+    environment:
+      - GF_PATHS_PROVISIONING=/etc/grafana/provisioning
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
     volumes:
-      - ./pg_data:/var/lib/postgresql/data
-    networks:
-      internal:
-        ipv4_address: 172.16.239.3
+      - ./grafana_datasources.yaml:/etc/grafana/provisioning/datasources/ds.yaml
+    ports:
+      - "3000:3000"
 
-networks:
-  internal:
-    driver: bridge
-    ipam:
-      driver: default
-      config:
-        - subnet: 172.16.239.0/24
-          gateway: 172.16.239.1
+  tempo:
+    image: grafana/tempo:latest
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ./tempo.yaml:/etc/tempo.yaml
+    ports:
+      - "4318:4318"
+
+  pyroscope:
+    image: grafana/pyroscope
+    ports:
+      - "4040:4040"

redis/jobs/consumers/utils.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"log/slog"
 	"net/http"
+	"time"
 
 	"github.com/Vilsol/slox"
 	"github.com/pkg/errors"
@@ -19,6 +20,7 @@ import (
 func UpdateModDataFromStorage(ctx context.Context, modID string, versionID string, metadata bool) error {
 	// perform task
 	slox.Info(ctx, "Updating DB for mod version with metadata", slog.String("mod", modID), slog.String("version", versionID), slog.Bool("metadata", metadata))
+	start := time.Now()
 
 	version, err := db.From(ctx).Version.Get(ctx, versionID)
 	if err != nil {
@@ -45,7 +47,7 @@ func UpdateModDataFromStorage(ctx context.Context, modID string, versionID strin
 
 	info, err := validation.ExtractModInfo(ctx, fileData, metadata, false, mod.ModReference)
 	if err != nil {
-		slox.Warn(ctx, "failed updating mod, likely outdated", slog.Any("err", err), slog.String("version", versionID))
+		slox.Warn(ctx, "failed updating mod, likely outdated", slog.Any("err", err), slog.String("version", versionID), slog.String("link", link), slog.Int("size", len(fileData)), slog.String("mod_reference", mod.ModReference))
 		// Outdated version
 		return nil
 	}
@@ -91,7 +93,7 @@ func UpdateModDataFromStorage(ctx context.Context, modID string, versionID strin
 	versionMinor := int(info.Semver.Minor())
 	versionPatch := int(info.Semver.Patch())
 
-	return version.Update().
+	err = version.Update().
 		SetSize(info.Size).
 		SetHash(info.Hash).
 		SetVersionMajor(versionMajor).
@@ -100,4 +102,11 @@ func UpdateModDataFromStorage(ctx context.Context, modID string, versionID strin
 		SetModReference(info.ModReference).
 		SetGameVersion(info.GameVersion).
 		Exec(ctx)
+	if err != nil {
+		return err
+	}
+
+	slox.Info(ctx, "mod update success", slog.String("version", versionID), slog.String("mod_reference", mod.ModReference), slog.Duration("took", time.Since(start)))
+
+	return nil
 }