Various tooling improvements (#152)

* Build binary in Makefile with the right flags

* Fix Python integration tests in Makefile

* Fix gitignore for Python integration tests

* Refactor how integration tests are run

- remove extra Dockerfiles and use the existing one instead
- move docker-compose.yml to the root of the repository
- cache building of the Docker image
- do not clean the services when stopping them, but instead in a
  separate make target
- run integration tests as a mounted volume instead of copying them
  into the container
- introduce helpers to get the environment variables for the
  integration tests and provide defaults for them
- add BuildX to CI

* Add Docker in Docker to devcontainer

* Rename source dir to fasttrackml within devcontainer

* Default to debug log level in devcontainer

.devcontainer/.env

@@ -2,4 +2,5 @@ POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
 POSTGRES_DB=postgres
 POSTGRES_HOSTNAME=localhost
-LC_COLLATE=POSIX
+LC_COLLATE=POSIX
+FML_LOG_LEVEL=DEBUG

.devcontainer/devcontainer.json

@@ -4,7 +4,7 @@
 	"name": "Fasttrack",
 	"dockerComposeFile": "docker-compose.yml",
 	"service": "app",
-	"workspaceFolder": "/workspaces/fasttrack",
+	"workspaceFolder": "/workspaces/fasttrackml",
 	// Configure tool-specific properties.
 	"customizations": {
 		// Configure properties specific to VS Code.
@@ -15,7 +15,7 @@
 				"go.useLanguageServer": true,
 				"go.gopath": "/go",
 				"go.goroot": "/usr/local/go",
-				"terminal.integrated.defaultProfile.linux": "zsh",
+				"terminal.integrated.defaultProfile.linux": "zsh"
 			},
 			// Add the IDs of extensions you want installed when the container is created.
 			"extensions": [
@@ -36,6 +36,9 @@
 	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode",
 	"features": {
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {
+			"dockerDashComposeVersion": "v2"
+		},
 		"ghcr.io/devcontainers/features/git:1": {},
 		"ghcr.io/devcontainers/features/github-cli:1": {},
 		"ghcr.io/dhoeric/features/k6:1": {}

.devcontainer/docker-compose.yml

@@ -31,7 +31,7 @@ services:
     volumes:
       - home:/home/vscode
       - workspaces:/workspaces
-      - ..:/workspaces/fasttrack:cached
+      - ..:/workspaces/fasttrackml:cached
 
     # Overrides default command so things don't shut down after the process ends.
     command: sleep infinity

.github/workflows/ci.yml

@@ -69,19 +69,25 @@ jobs:
         run: make test-go-unit
 
   golang-integration-tests:
+    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
     name: Golang Integration Tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
           go-version: "1.20"
 
       - name: Run Integration Tests
         run: make service-test
+        env:
+          DOCKER_BUILDKIT: 1
 
   python-integration-tests:
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id

.gitignore

@@ -9,7 +9,7 @@ fasttrackml
 fasttrackml.db*
 
 # Imported integration tests
-tests/*/*.src
+tests/integration/python/*/*.src
 
 # Quick start
 outputs

.vscode/launch.json

@@ -12,10 +12,8 @@
             "program": "${workspaceFolder}",
             "args": [
                 "server",
-                "--log-level",
-                "debug",
                 "--database-uri",
-                "postgres://postgres:postgres@localhost:5432/postgres"
+                "postgres://postgres:postgres@localhost/postgres"
             ]
         },
         {
@@ -26,8 +24,6 @@
             "program": "${workspaceFolder}",
             "args": [
                 "server",
-                "--log-level",
-                "debug",
                 "--database-uri",
                 "sqlite://fasttrackml.db?mode=memory&cache=shared"
             ]
@@ -40,8 +36,6 @@
             "program": "${workspaceFolder}",
             "args": [
                 "server",
-                "--log-level",
-                "debug",
                 "--database-uri",
                 "sqlite://fasttrackml.db"
             ]
@@ -54,8 +48,6 @@
             "program": "${workspaceFolder}",
             "args": [
                 "server",
-                "--log-level",
-                "debug",
                 "--database-uri",
                 "sqlite://encrypted.db?_key=passphrase"
             ]

Makefile

@@ -4,21 +4,12 @@
 #
 # Project-specific variables
 #
-# Service name. Used for binary name, docker-compose service name, etc...
-SERVICE=fasttrack-service
+# App name.
+APP=fml
 # Enable Go Modules.
 GO111MODULE=on
 
 #
-# General variables
-#
-# Path to Docker file
-PATH_DOCKER_FILE=$(realpath ./docker/Dockerfile)
-# Path to docker-compose file
-PATH_DOCKER_COMPOSE_FILE=$(realpath ./docker/docker-compose.yml)
-# Docker compose starting options.
-DOCKER_COMPOSE_OPTIONS= -f $(PATH_DOCKER_COMPOSE_FILE)
-
 # Default target (help)
 #
 .PHONY: help
@@ -43,9 +34,9 @@ go-get: ## get go modules.
 	@go mod download
 
 .PHONY: go-build
-go-build: ## build service binary.
+go-build: ## build app binary.
 	@echo '>>> Building go binary.'
-	@go build -ldflags="-s -w" -o $(SERVICE) ./main.go
+	@go build -ldflags="-linkmode external -extldflags -static -s -w" -tags "$$(jq -r '."go.buildTags"' .vscode/settings.json)" -o $(APP) ./main.go
 
 #
 # Tests targets.
@@ -61,45 +52,58 @@ test-go-integration: ## run go integration tests.
 	go test -v -p 1 -tags="integration" ./tests/integration/golang/...
 
 PHONY: test-python-integration
-test-python-integration: build ## run the MLFlow python integration tests.
+test-python-integration: test-python-integration-mlflow test-python-integration-aim  ## run all the python integration tests.
+
+PHONY: test-python-integration-mlflow
+test-python-integration-mlflow: build ## run the MLFlow python integration tests.
 	@echo ">>> Running MLFlow python integration tests."
-	tests/mlflow/test.sh
+	tests/integration/python/mlflow/test.sh
+
+PHONY: test-python-integration-aim
+test-python-integration-aim: build ## run the Aim python integration tests.
+	@echo ">>> Running Aim python integration tests."
+	tests/integration/python/aim/test.sh
 
 #
 # Service test targets
 #
 .PHONY: service-build
-service-build: ## build service and all it's dependencies
-	@docker-compose $(DOCKER_COMPOSE_OPTIONS) build --no-cache
+service-build: ## build service and all its dependencies
+	@docker-compose build
 
 .PHONY: start-service-dependencies
 service-start-dependencies: ## start service dependencies in docker.
 	@echo ">>> Start all Service dependencies."
-	@docker-compose $(DOCKER_COMPOSE_OPTIONS) up \
+	@docker-compose up \
 	-d \
-	fasttrack-postgres
+	postgres
 
 .PHONY: service-start
 service-start: service-build service-start-dependencies ## start service in docker.
 	@echo ">>> Sleeping 5 seconds until dependencies start."
 	@sleep 5
 	@echo ">>> Starting service."
 	@echo ">>> Starting up service container."
-	@docker-compose $(DOCKER_COMPOSE_OPTIONS) up -d $(SERVICE)
+	@docker-compose up -d service
 
 .PHONY: service-stop
 service-stop: ## stop service in docker.
 	@echo ">>> Stopping service."
-	@docker-compose $(DOCKER_COMPOSE_OPTIONS) down -v --remove-orphans
+	@docker-compose stop
 
 .PHONY: service-restart
 service-restart: service-stop service-start ## restart service in docker
 
 .PHONY: service-test
 service-test: service-stop service-start ## run tests over the service in docker.
 	@echo ">>> Running tests over service."
-	@docker-compose $(DOCKER_COMPOSE_OPTIONS) \
-		run fasttrack-integration-tests
+	@docker-compose \
+		run integration-tests
+
+.PHONY: service-clean
+service-clean: ## clean service in docker.
+	@echo ">>> Cleaning service."
+	@docker-compose down -v --remove-orphans
 
 #
 # Mockery targets.
@@ -126,4 +130,4 @@ build: go-build ## build the go components
 PHONY: run
 run: build ## run the FastTrackML server
 	@echo ">>> Running the FasttrackML server."
-	./$(SERVICE) server
+	./$(APP) server

docker-compose.yml

@@ -0,0 +1,33 @@
+services:
+  service:
+    build:
+      context: .
+      args:
+        tags: netgo osusergo sqlite_foreign_keys sqlite_math_functions sqlite_omit_load_extension sqlite_unlock_notify sqlite_vacuum_incr
+    depends_on:
+      - postgres
+    environment:
+      FML_DATABASE_URI: postgres://postgres:postgres@postgres/postgres
+      FML_LOG_LEVEL: debug
+
+  postgres:
+    image: postgres:latest
+    environment:
+      - POSTGRES_PASSWORD=postgres
+
+  integration-tests:
+    image: golang:1.20
+    command: make test-go-integration
+    volumes:
+      - .:/go/src
+      - go-cache:/go/pkg
+    working_dir: /go/src
+    depends_on:
+      - service
+      - postgres
+    environment:
+      FML_DATABASE_URI: postgres://postgres:postgres@postgres/postgres
+      FML_SERVICE_URI: http://service:5000
+
+volumes:
+  go-cache:

tests/integration/golang/aim/metric/search_test.go

@@ -7,7 +7,6 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"os"
 	"testing"
 
 	"github.com/google/uuid"
@@ -32,12 +31,12 @@ func TestSearchMetricsTestSuite(t *testing.T) {
 }
 
 func (s *SearchMetricsTestSuite) SetupTest() {
-	s.client = helpers.NewAimApiClient(os.Getenv("SERVICE_BASE_URL"))
-	runFixtures, err := fixtures.NewRunFixtures(os.Getenv("DATABASE_DSN"))
+	s.client = helpers.NewAimApiClient(helpers.GetServiceUri())
+	runFixtures, err := fixtures.NewRunFixtures(helpers.GetDatabaseUri())
 	assert.Nil(s.T(), err)
-	metricFixtures, err := fixtures.NewMetricFixtures(os.Getenv("DATABASE_DSN"))
+	metricFixtures, err := fixtures.NewMetricFixtures(helpers.GetDatabaseUri())
 	assert.Nil(s.T(), err)
-	experimentFixtures, err := fixtures.NewExperimentFixtures(os.Getenv("DATABASE_DSN"))
+	experimentFixtures, err := fixtures.NewExperimentFixtures(helpers.GetDatabaseUri())
 	assert.Nil(s.T(), err)
 	s.experimentFixtures = experimentFixtures
 

tests/integration/golang/aim/run/archive_batch_test.go

@@ -5,7 +5,6 @@ package run
 import (
 	"context"
 	"fmt"
-	"os"
 	"testing"
 
 	"github.com/gofiber/fiber/v2"
@@ -31,11 +30,11 @@ func TestArchiveBatchTestSuite(t *testing.T) {
 }
 
 func (s *ArchiveBatchTestSuite) SetupTest() {
-	s.client = helpers.NewAimApiClient(os.Getenv("SERVICE_BASE_URL"))
-	runFixtures, err := fixtures.NewRunFixtures(os.Getenv("DATABASE_DSN"))
+	s.client = helpers.NewAimApiClient(helpers.GetServiceUri())
+	runFixtures, err := fixtures.NewRunFixtures(helpers.GetDatabaseUri())
 	assert.Nil(s.T(), err)
 	s.runFixtures = runFixtures
-	expFixtures, err := fixtures.NewExperimentFixtures(os.Getenv("DATABASE_DSN"))
+	expFixtures, err := fixtures.NewExperimentFixtures(helpers.GetDatabaseUri())
 	assert.Nil(s.T(), err)
 	s.experimentFixtures = expFixtures
 
@@ -109,7 +108,6 @@ func (s *ArchiveBatchTestSuite) Test_Ok() {
 			assert.Nil(s.T(), err)
 			assert.Equal(s.T(), originalMinRowNum, newMinRowNum)
 			assert.Equal(s.T(), originalMaxRowNum, newMaxRowNum)
-
 		})
 	}
 }