Installation: Day 1 improvements to settings

.gitignore

@@ -175,6 +175,6 @@ cython_debug/
 *.db
 *.db-journal
 
-# env diff files
-.env.diff
-.env.diff.new
+# Deployment/config files
+build/
+config.user.yaml

Makefile

@@ -1,11 +1,24 @@
-.PHONY: local-up local-down local-logs clean-docker-buildcache clean-docker-images clean-docker-containers start-lumigator-external-services start-lumigator start-lumigator-postgres stop-lumigator test-sdk-unit test-sdk-integration test-sdk-integration-containers test-sdk test-backend-unit test-backend-integration test-backend-integration-containers test-backend test-jobs-evaluation-unit test-jobs-inference-unit test-jobs test-all check-dot-env
+.PHONY: local-up local-down local-logs clean-docker-buildcache clean-docker-images clean-docker-containers start-lumigator-external-services start-lumigator start-lumigator-postgres stop-lumigator test-sdk-unit test-sdk-integration test-sdk-integration-containers test-sdk test-backend-unit test-backend-integration test-backend-integration-containers test-backend test-jobs-evaluation-unit test-jobs-inference-unit test-jobs test-all config-clean config-generate config-generate-env
 
 SHELL:=/bin/bash
 UNAME:= $(shell uname -o)
+
+# Required binaries in order to correctly run the makefile, if any cannot be found the script will fail
+REQUIRED_BINARIES := git docker yq uv
+$(foreach bin,$(REQUIRED_BINARIES),\
+  $(if $(shell command -v $(bin) 2> /dev/null),,$(error Please install `$(bin)`)))
+
 PROJECT_ROOT := $(shell git rev-parse --show-toplevel)
 CONTAINERS_RUNNING := $(shell docker ps -q --filter "name=lumigator-")
 
-KEEP_CONTAINERS_UP := $(shell grep -E '^KEEP_CONTAINERS_UP=' .env | cut -d'=' -f2 | tr -d '"' || echo "FALSE")
+KEEP_CONTAINERS_UP ?= "FALSE"
+
+# Configuration to identify the input and output config files
+# NOTE: Changing CONFIG_BUILD_DIR will require review of .gitignore
+CONFIG_BUILD_DIR=build
+CONFIG_DEFAULT=config.default.yaml
+CONFIG_USER=config.user.yaml
+CONFIG_OUTPUT=config.deploy.yaml
 
 # used in docker-compose to choose the right Ray image
 ARCH := $(shell uname -m)
@@ -87,62 +100,51 @@ LOCAL_DOCKERCOMPOSE_FILE:= docker-compose.yaml
 DEV_DOCKER_COMPOSE_FILE:= .devcontainer/docker-compose.override.yaml
 POSTGRES_DOCKER_COMPOSE_FILE:= .devcontainer/docker-compose-postgres.override.yaml
 
-check-dot-env:
-#    Create .env from template if it doesn't exist
-	@if [ ! -f .env ]; then \
-	cp .env.template .env; \
-	echo ".env created from .env.template"; \
-	fi
-
-	# Generate new diff between template and current .env
-	@diff .env.template .env > .env.diff.new 2>/dev/null || true
-
-	# Check if files are out of sync and show warning
-	@if [ -f .env ] && [ -f .env.template ] && ! cmp -s .env.diff .env.diff.new; then \
-	echo -e "\033[1;31m====================================================================\033[0m"; \
-	echo -e "\033[1;31mWARNING: .env and .env.template are out of sync. Please review changes\033[0m"; \
-	echo -e "\033[1;31m====================================================================\033[0m"; \
-	fi
-
-	# Update diff file for next comparison
-	@mv .env.diff.new .env.diff 2>/dev/null || true
+define remove_config_dir
+	@echo "Cleaning up config build directory: '$(CONFIG_BUILD_DIR)'..."
+	@rm -rf $(CONFIG_BUILD_DIR)
+	@echo "Cleanup complete."
+endef
 
 # Launches Lumigator in 'development' mode (all services running locally, code mounted in)
-local-up: check-dot-env
+local-up: config-generate-env
 	uv run pre-commit install
-	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) -f ${DEV_DOCKER_COMPOSE_FILE} up --watch --build
+	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --env-file "$(CONFIG_BUILD_DIR)/.env" --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) -f ${DEV_DOCKER_COMPOSE_FILE} up --watch --build
 
 local-down:
-	docker compose --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) -f ${DEV_DOCKER_COMPOSE_FILE} down
+	docker compose --env-file "$(CONFIG_BUILD_DIR)/.env" --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) -f ${DEV_DOCKER_COMPOSE_FILE} down
+	$(call remove_config_dir)
 
 local-logs:
 	docker compose -f $(LOCAL_DOCKERCOMPOSE_FILE) logs
 
 # Launches lumigator in 'user-local' mode (All services running locally, using latest docker container, no code mounted in) - postgres version
-start-lumigator-postgres: check-dot-env
-	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) -f $(POSTGRES_DOCKER_COMPOSE_FILE) up -d
+start-lumigator-postgres: config-generate-env
+	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --env-file "$(CONFIG_BUILD_DIR)/.env" --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) -f $(POSTGRES_DOCKER_COMPOSE_FILE) up -d
 
 stop-lumigator-postgres:
-	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) -f $(POSTGRES_DOCKER_COMPOSE_FILE) down
+	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --env-file "$(CONFIG_BUILD_DIR)/.env" --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) -f $(POSTGRES_DOCKER_COMPOSE_FILE) down
+	$(call remove_config_dir)
 
 # Launches lumigator in 'user-local' mode (All services running locally, using latest docker container, no code mounted in)
-start-lumigator: check-dot-env
-	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) up -d
+start-lumigator: config-generate-env
+	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --env-file "$(CONFIG_BUILD_DIR)/.env" --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) up -d
 
 # Launches lumigator with no code mounted in, and forces build of containers (used in CI for integration tests)
-start-lumigator-build: check-dot-env
-	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) up -d --build
+start-lumigator-build: config-generate-env
+	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --env-file "$(CONFIG_BUILD_DIR)/.env" --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) up -d --build
 
 # Launches lumigator with no code mounted in, and forces build of containers (used in CI for integration tests)
-start-lumigator-build-postgres: check-dot-env
-	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) -f $(POSTGRES_DOCKER_COMPOSE_FILE) up -d --build
+start-lumigator-build-postgres: config-generate-env
+	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --env-file "$(CONFIG_BUILD_DIR)/.env" --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) -f $(POSTGRES_DOCKER_COMPOSE_FILE) up -d --build
 
 # Launches lumigator without local dependencies (ray, S3)
-start-lumigator-external-services: check-dot-env
-	docker compose $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) up -d
+start-lumigator-external-services: config-generate-env
+	docker compose --env-file "$(CONFIG_BUILD_DIR)/.env"$(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) up -d
 
 stop-lumigator:
-	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) down
+	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --env-file "$(CONFIG_BUILD_DIR)/.env" --profile local $(GPU_COMPOSE) -f $(LOCAL_DOCKERCOMPOSE_FILE) down
+	$(call remove_config_dir)
 
 clean-docker-buildcache:
 	docker builder prune --all -f
@@ -157,7 +159,7 @@ clean-docker-images:
 
 clean-docker-all: clean-docker-containers clean-docker-buildcache clean-docker-images
 
-clean-all: clean-docker-buildcache clean-docker-containers
+clean-all: clean-docker-buildcache clean-docker-containers config-clean
 
 
 # SDK tests
@@ -244,3 +246,28 @@ test-jobs-unit: test-jobs-evaluation-unit test-jobs-inference-unit
 
 # test everything
 test-all: test-sdk test-backend test-jobs-unit
+
+# config-clean: removes any generated config files from the build directory (including the directory itself).
+config-clean:
+	$(call remove_config_dir)
+
+# config-generate: merges default and user YAML config files to produce a YAML file in the build directory.
+# Any keys missing from the user config will be substituted with the default key/values.
+# Values in the user config take prescedence over the default values.
+config-generate: config-clean
+	@mkdir -p $(CONFIG_BUILD_DIR)
+	@if [ -e $(CONFIG_USER) ]; then \
+		echo "Found user configuration"; \
+		yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' $(CONFIG_DEFAULT) $(CONFIG_USER) > $(CONFIG_BUILD_DIR)/$(CONFIG_OUTPUT); \
+	else \
+		echo "No user defined config found, default will be used"; \
+		cp $(CONFIG_DEFAULT) $(CONFIG_USER); \
+		cp $(CONFIG_DEFAULT) $(CONFIG_BUILD_DIR)/$(CONFIG_OUTPUT); \
+	fi
+	@echo "Config generation complete."
+
+# config-generate-env: parses a generated config YAML file and outputs a .env file ready for use in Docker.
+config-generate-env: config-generate
+	@echo "Generating .env file for Docker Compose using 'app' config section..."
+	@scripts/config_generate_env.sh $(CONFIG_BUILD_DIR)/$(CONFIG_OUTPUT) "$(CONFIG_BUILD_DIR)/.env"
+	@echo ".env file generated at '$(CONFIG_BUILD_DIR)/.env'"

config.default.yaml

@@ -0,0 +1,54 @@
+app:
+    # AWS Variables for S3 Object Storage
+    # Configure these for AWS access, or use defaults for local development with minio.
+    aws_access_key_id: lumigator
+    aws_secret_access_key: lumigator # pragma: allowlist secret
+    aws_default_region: us-east-2
+    # The URL which can be used for S3 operations
+    aws_endpoint_url: http://localhost:9000
+    # The name of the S3 bucket to use for storage
+    s3_bucket: lumigator-storage
+    # Ray Cluster Configuration
+    # These settings are for the local Ray setup.
+    # To use an external Ray cluster, you MUST use an external S3-compatible storage
+    # to ensure the Ray workers can access data from your Lumigator server.
+    ray_head_node_host: ray
+    ray_dashboard_port: 8265
+    ray_worker_gpus: 0
+    ray_worker_gpus_fraction: 0
+    nvidia_visible_devices: all
+    gpu_count: 0
+    # Cache for HuggingFace models and artifacts ($ escapes resolution during deployment)
+    hf_home: $${HOME}/.cache/huggingface
+    # Access token to use when attempting to access gated models in HuggingFace
+    hf_token:
+    # Access token to use when attempting to interact with Mistral's API
+    mistral_api_key:
+    # Access token to use when attempting to interact with OpenAI's API
+    openai_api_key:
+    # MLFlow Configuration
+    mlflow_tracking_uri: http://mlflow:5000
+    mlflow_database_url: sqlite:///mlflow.db
+    mlflow_s3_root_path: s3://mlflow
+    # S3 Configuration (MinIO)
+    minio_root_user: minioadmin
+    minio_root_password: minioadmin # pragma: allowlist secret
+    minio_api_cors_allow_origin: "*"
+    deployment_type: local
+    database_url: sqlite:///local.db
+    # lumigator_api_cors_allowed_origins:
+    # Array of origins (See: https://developer.mozilla.org/en-US/docs/Glossary/Origin)
+    # that should be allowed to make Cross-Domain (CORS) API requests to the Lumigator backend API.
+    # The expected format of each is: scheme + domain + port (if no port is specified then 80 is assumed).
+    # e.g. "http://localhost:3000,http://lumigator.mydomain.com"
+    # To allow CORS requests from anywhere specify "*" as any, or the only value.
+    # e.g.  - "*"
+    lumigator_api_cors_allowed_origins:
+        - http://localhost
+        - http://localhost:3000
+    evaluator_pip_reqs: /mzai/lumigator/jobs/evaluator/requirements.txt
+    evaluator_work_dir: /mzai/lumigator/jobs/evaluator
+    evaluator_lite_pip_reqs: /mzai/lumigator/jobs/evaluator_lite/requirements.txt
+    evaluator_lite_work_dir: /mzai/lumigator/jobs/evaluator_lite
+    inference_pip_reqs: /mzai/lumigator/jobs/inference/requirements.txt
+    inference_work_dir: /mzai/lumigator/jobs/inference

docker-compose.gpu.override.yaml

@@ -13,9 +13,9 @@ services:
               count: $GPU_COUNT
               capabilities: [gpu]
     environment:
-      - NVIDIA_VISIBLE_DEVICES=$NVIDIA_VISIBLE_DEVICES
+      - NVIDIA_VISIBLE_DEVICES
 
   backend:
     environment:
-      - RAY_WORKER_GPUS=$RAY_WORKER_GPUS
-      - RAY_WORKER_GPUS_FRACTION=$RAY_WORKER_GPUS_FRACTION
+      - RAY_WORKER_GPUS
+      - RAY_WORKER_GPUS_FRACTION