diff --git a/.github/workflows/platform-autogpt-deploy.yaml b/.github/workflows/platform-autogpt-deploy.yaml new file mode 100644 index 000000000000..97c2fe78749e --- /dev/null +++ b/.github/workflows/platform-autogpt-deploy.yaml @@ -0,0 +1,152 @@ +name: AutoGPT Platform - Build, Push, and Deploy Dev Environment + +on: + push: + branches: [ dev ] + paths: + - 'autogpt_platform/backend/**' + - 'autogpt_platform/frontend/**' + - 'autogpt_platform/market/**' + +permissions: + contents: 'read' + id-token: 'write' + +env: + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + GKE_CLUSTER: dev-gke-cluster + GKE_ZONE: us-central1-a + NAMESPACE: dev-agpt + +jobs: + build-push-deploy: + name: Build, Push, and Deploy + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - id: 'auth' + uses: 'google-github-actions/auth@v1' + with: + workload_identity_provider: 'projects/638488734936/locations/global/workloadIdentityPools/dev-pool/providers/github' + service_account: 'dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com' + token_format: 'access_token' + create_credentials_file: true + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + + - name: 'Configure Docker' + run: | + gcloud auth configure-docker us-east1-docker.pkg.dev + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Check for changes + id: check_changes + run: | + git fetch origin dev + BACKEND_CHANGED=$(git diff --name-only origin/dev HEAD | grep "^autogpt_platform/backend/" && echo "true" || echo "false") + FRONTEND_CHANGED=$(git diff --name-only origin/dev HEAD | grep "^autogpt_platform/frontend/" && echo "true" || echo "false") + MARKET_CHANGED=$(git diff --name-only origin/dev HEAD | grep "^autogpt_platform/market/" && echo "true" || echo "false") + echo "backend_changed=$BACKEND_CHANGED" >> $GITHUB_OUTPUT + echo "frontend_changed=$FRONTEND_CHANGED" >> $GITHUB_OUTPUT + echo "market_changed=$MARKET_CHANGED" >> $GITHUB_OUTPUT + + - name: Get GKE credentials + uses: 'google-github-actions/get-gke-credentials@v1' + with: + cluster_name: ${{ env.GKE_CLUSTER }} + location: ${{ env.GKE_ZONE }} + + - name: Build and Push Backend + if: steps.check_changes.outputs.backend_changed == 'true' + uses: docker/build-push-action@v2 + with: + context: . + file: ./autogpt_platform/backend/Dockerfile + push: true + tags: us-east1-docker.pkg.dev/agpt-dev/agpt-backend-dev/agpt-backend-dev:${{ github.sha }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + - name: Build and Push Frontend + if: steps.check_changes.outputs.frontend_changed == 'true' + uses: docker/build-push-action@v2 + with: + context: . + file: ./autogpt_platform/frontend/Dockerfile + push: true + tags: us-east1-docker.pkg.dev/agpt-dev/agpt-frontend-dev/agpt-frontend-dev:${{ github.sha }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + - name: Build and Push Market + if: steps.check_changes.outputs.market_changed == 'true' + uses: docker/build-push-action@v2 + with: + context: . + file: ./autogpt_platform/market/Dockerfile + push: true + tags: us-east1-docker.pkg.dev/agpt-dev/agpt-market-dev/agpt-market-dev:${{ github.sha }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + - name: Set up Helm + uses: azure/setup-helm@v1 + with: + version: v3.4.0 + + - name: Deploy Backend + if: steps.check_changes.outputs.backend_changed == 'true' + run: | + helm upgrade autogpt-server ./autogpt-server \ + --namespace ${{ env.NAMESPACE }} \ + -f autogpt-server/values.yaml \ + -f autogpt-server/values.dev.yaml \ + --set image.tag=${{ github.sha }} + + - name: Deploy Websocket + if: steps.check_changes.outputs.backend_changed == 'true' + run: | + helm upgrade autogpt-websocket-server ./autogpt-websocket-server \ + --namespace ${{ env.NAMESPACE }} \ + -f autogpt-websocket-server/values.yaml \ + -f autogpt-websocket-server/values.dev.yaml \ + --set image.tag=${{ github.sha }} + + - name: Deploy Market + if: steps.check_changes.outputs.market_changed == 'true' + run: | + helm upgrade autogpt-market ./autogpt-market \ + --namespace ${{ env.NAMESPACE }} \ + -f autogpt-market/values.yaml \ + -f autogpt-market/values.dev.yaml \ + --set image.tag=${{ github.sha }} + + - name: Deploy Frontend + if: steps.check_changes.outputs.frontend_changed == 'true' + run: | + helm upgrade autogpt-builder ./autogpt-builder \ + --namespace ${{ env.NAMESPACE }} \ + -f autogpt-builder/values.yaml \ + -f autogpt-builder/values.dev.yaml \ + --set image.tag=${{ github.sha }} \ No newline at end of file diff --git a/.github/workflows/platform-frontend-ci.yml b/.github/workflows/platform-frontend-ci.yml index 4262cd6484ad..ce3633013bb9 100644 --- a/.github/workflows/platform-frontend-ci.yml +++ b/.github/workflows/platform-frontend-ci.yml @@ -39,10 +39,27 @@ jobs: runs-on: ubuntu-latest steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: false + dotnet: false + haskell: false + large-packages: true + docker-images: true + swap-storage: true + - name: Checkout repository uses: actions/checkout@v4 with: submodules: recursive + - name: Set up Node.js uses: actions/setup-node@v4 with: diff --git a/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/store.py b/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/store.py index 44cc9b60f4a1..f4ce921937e7 100644 --- a/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/store.py +++ b/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/store.py @@ -1,10 +1,10 @@ import secrets from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING if TYPE_CHECKING: from redis import Redis - from supabase import Client + from backend.executor.database import DatabaseManager from autogpt_libs.utils.synchronize import RedisKeyedMutex @@ -18,8 +18,8 @@ class SupabaseIntegrationCredentialsStore: - def __init__(self, supabase: "Client", redis: "Redis"): - self.supabase = supabase + def __init__(self, redis: "Redis", db: "DatabaseManager"): + self.db_manager: DatabaseManager = db self.locks = RedisKeyedMutex(redis) def add_creds(self, user_id: str, credentials: Credentials) -> None: @@ -35,7 +35,9 @@ def add_creds(self, user_id: str, credentials: Credentials) -> None: def get_all_creds(self, user_id: str) -> list[Credentials]: user_metadata = self._get_user_metadata(user_id) - return UserMetadata.model_validate(user_metadata).integration_credentials + return UserMetadata.model_validate( + user_metadata.model_dump() + ).integration_credentials def get_creds_by_id(self, user_id: str, credentials_id: str) -> Credentials | None: all_credentials = self.get_all_creds(user_id) @@ -90,9 +92,7 @@ def delete_creds_by_id(self, user_id: str, credentials_id: str) -> None: ] self._set_user_integration_creds(user_id, filtered_credentials) - async def store_state_token( - self, user_id: str, provider: str, scopes: list[str] - ) -> str: + def store_state_token(self, user_id: str, provider: str, scopes: list[str]) -> str: token = secrets.token_urlsafe(32) expires_at = datetime.now(timezone.utc) + timedelta(minutes=10) @@ -105,17 +105,17 @@ async def store_state_token( with self.locked_user_metadata(user_id): user_metadata = self._get_user_metadata(user_id) - oauth_states = user_metadata.get("integration_oauth_states", []) + oauth_states = user_metadata.integration_oauth_states oauth_states.append(state.model_dump()) - user_metadata["integration_oauth_states"] = oauth_states + user_metadata.integration_oauth_states = oauth_states - self.supabase.auth.admin.update_user_by_id( - user_id, {"user_metadata": user_metadata} + self.db_manager.update_user_metadata( + user_id=user_id, metadata=user_metadata ) return token - async def get_any_valid_scopes_from_state_token( + def get_any_valid_scopes_from_state_token( self, user_id: str, token: str, provider: str ) -> list[str]: """ @@ -126,7 +126,7 @@ async def get_any_valid_scopes_from_state_token( THE CODE FOR TOKENS. """ user_metadata = self._get_user_metadata(user_id) - oauth_states = user_metadata.get("integration_oauth_states", []) + oauth_states = user_metadata.integration_oauth_states now = datetime.now(timezone.utc) valid_state = next( @@ -145,10 +145,10 @@ async def get_any_valid_scopes_from_state_token( return [] - async def verify_state_token(self, user_id: str, token: str, provider: str) -> bool: + def verify_state_token(self, user_id: str, token: str, provider: str) -> bool: with self.locked_user_metadata(user_id): user_metadata = self._get_user_metadata(user_id) - oauth_states = user_metadata.get("integration_oauth_states", []) + oauth_states = user_metadata.integration_oauth_states now = datetime.now(timezone.utc) valid_state = next( @@ -165,10 +165,8 @@ async def verify_state_token(self, user_id: str, token: str, provider: str) -> b if valid_state: # Remove the used state oauth_states.remove(valid_state) - user_metadata["integration_oauth_states"] = oauth_states - self.supabase.auth.admin.update_user_by_id( - user_id, {"user_metadata": user_metadata} - ) + user_metadata.integration_oauth_states = oauth_states + self.db_manager.update_user_metadata(user_id, user_metadata) return True return False @@ -177,19 +175,13 @@ def _set_user_integration_creds( self, user_id: str, credentials: list[Credentials] ) -> None: raw_metadata = self._get_user_metadata(user_id) - raw_metadata.update( - {"integration_credentials": [c.model_dump() for c in credentials]} - ) - self.supabase.auth.admin.update_user_by_id( - user_id, {"user_metadata": raw_metadata} - ) + raw_metadata.integration_credentials = [c.model_dump() for c in credentials] + self.db_manager.update_user_metadata(user_id, raw_metadata) def _get_user_metadata(self, user_id: str) -> UserMetadataRaw: - response = self.supabase.auth.admin.get_user_by_id(user_id) - if not response.user: - raise ValueError(f"User with ID {user_id} not found") - return cast(UserMetadataRaw, response.user.user_metadata) + metadata: UserMetadataRaw = self.db_manager.get_user_metadata(user_id=user_id) + return metadata def locked_user_metadata(self, user_id: str): - key = (self.supabase.supabase_url, f"user:{user_id}", "metadata") + key = (self.db_manager, f"user:{user_id}", "metadata") return self.locks.locked(key) diff --git a/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/types.py b/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/types.py index da39f6a842c2..0f973bb52484 100644 --- a/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/types.py +++ b/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/types.py @@ -56,6 +56,7 @@ class OAuthState(BaseModel): token: str provider: str expires_at: int + scopes: list[str] """Unix timestamp (seconds) indicating when this OAuth state expires""" @@ -64,6 +65,6 @@ class UserMetadata(BaseModel): integration_oauth_states: list[OAuthState] = Field(default_factory=list) -class UserMetadataRaw(TypedDict, total=False): - integration_credentials: list[dict] - integration_oauth_states: list[dict] +class UserMetadataRaw(BaseModel): + integration_credentials: list[dict] = Field(default_factory=list) + integration_oauth_states: list[dict] = Field(default_factory=list) diff --git a/autogpt_platform/backend/Dockerfile b/autogpt_platform/backend/Dockerfile index f697db11982c..5795398d1fae 100644 --- a/autogpt_platform/backend/Dockerfile +++ b/autogpt_platform/backend/Dockerfile @@ -8,7 +8,7 @@ WORKDIR /app # Install build dependencies RUN apt-get update \ - && apt-get install -y build-essential curl ffmpeg wget libcurl4-gnutls-dev libexpat1-dev gettext libz-dev libssl-dev postgresql-client git \ + && apt-get install -y build-essential curl ffmpeg wget libcurl4-gnutls-dev libexpat1-dev libpq5 gettext libz-dev libssl-dev postgresql-client git \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/autogpt_platform/backend/README.advanced.md b/autogpt_platform/backend/README.advanced.md index 829a3d79262a..09e0f90fcc25 100644 --- a/autogpt_platform/backend/README.advanced.md +++ b/autogpt_platform/backend/README.advanced.md @@ -37,7 +37,7 @@ We use the Poetry to manage the dependencies. To set up the project, follow thes 5. Generate the Prisma client ```sh - poetry run prisma generate --schema postgres/schema.prisma + poetry run prisma generate ``` @@ -61,7 +61,7 @@ We use the Poetry to manage the dependencies. To set up the project, follow thes ```sh cd ../backend - prisma migrate dev --schema postgres/schema.prisma + prisma migrate deploy ``` ## Running The Server diff --git a/autogpt_platform/backend/README.md b/autogpt_platform/backend/README.md index fc0c6b3944d0..00194b304e1c 100644 --- a/autogpt_platform/backend/README.md +++ b/autogpt_platform/backend/README.md @@ -59,7 +59,7 @@ We use the Poetry to manage the dependencies. To set up the project, follow thes ```sh docker compose up db redis -d - poetry run prisma migrate dev + poetry run prisma migrate deploy ``` ## Running The Server diff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.py index af69a9761abb..81a0839774d8 100644 --- a/autogpt_platform/backend/backend/blocks/llm.py +++ b/autogpt_platform/backend/backend/blocks/llm.py @@ -62,7 +62,7 @@ class LlmModel(str, Enum, metaclass=LlmModelMeta): GPT4_TURBO = "gpt-4-turbo" GPT3_5_TURBO = "gpt-3.5-turbo" # Anthropic models - CLAUDE_3_5_SONNET = "claude-3-5-sonnet-20240620" + CLAUDE_3_5_SONNET = "claude-3-5-sonnet-latest" CLAUDE_3_HAIKU = "claude-3-haiku-20240307" # Groq models LLAMA3_8B = "llama3-8b-8192" diff --git a/autogpt_platform/backend/backend/data/user.py b/autogpt_platform/backend/backend/data/user.py index db60eea235cb..6c8696379e37 100644 --- a/autogpt_platform/backend/backend/data/user.py +++ b/autogpt_platform/backend/backend/data/user.py @@ -1,6 +1,8 @@ from typing import Optional +from autogpt_libs.supabase_integration_credentials_store.types import UserMetadataRaw from fastapi import HTTPException +from prisma import Json from prisma.models import User from backend.data.db import prisma @@ -48,3 +50,21 @@ async def create_default_user(enable_auth: str) -> Optional[User]: ) return User.model_validate(user) return None + + +async def get_user_metadata(user_id: str) -> UserMetadataRaw: + user = await User.prisma().find_unique_or_raise( + where={"id": user_id}, + ) + return ( + UserMetadataRaw.model_validate(user.metadata) + if user.metadata + else UserMetadataRaw() + ) + + +async def update_user_metadata(user_id: str, metadata: UserMetadataRaw): + await User.prisma().update( + where={"id": user_id}, + data={"metadata": Json(metadata.model_dump())}, + ) diff --git a/autogpt_platform/backend/backend/executor/database.py b/autogpt_platform/backend/backend/executor/database.py index 8257d5c8c07c..aea2dd4177fc 100644 --- a/autogpt_platform/backend/backend/executor/database.py +++ b/autogpt_platform/backend/backend/executor/database.py @@ -16,6 +16,7 @@ ) from backend.data.graph import get_graph, get_node from backend.data.queue import RedisEventQueue +from backend.data.user import get_user_metadata, update_user_metadata from backend.util.service import AppService, expose from backend.util.settings import Config @@ -73,3 +74,7 @@ def wrapper(self, *args: P.args, **kwargs: P.kwargs) -> R: Callable[[Any, str, int, str, dict[str, str], float, float], int], exposed_run_and_wait(user_credit_model.spend_credits), ) + + # User + User Metadata + get_user_metadata = exposed_run_and_wait(get_user_metadata) + update_user_metadata = exposed_run_and_wait(update_user_metadata) diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py index dd6f6f75772b..ce6de9e9a623 100644 --- a/autogpt_platform/backend/backend/executor/manager.py +++ b/autogpt_platform/backend/backend/executor/manager.py @@ -31,7 +31,7 @@ from backend.data.model import CREDENTIALS_FIELD_NAME, CredentialsMetaInput from backend.integrations.creds_manager import IntegrationCredentialsManager from backend.util import json -from backend.util.cache import thread_cached_property +from backend.util.cache import thread_cached from backend.util.decorator import error_logged, time_measured from backend.util.logging import configure_logging from backend.util.process import set_service_name @@ -419,7 +419,7 @@ def on_node_executor_start(cls): redis.connect() cls.pid = os.getpid() cls.db_client = get_db_client() - cls.creds_manager = IntegrationCredentialsManager() + cls.creds_manager = IntegrationCredentialsManager(db_manager=cls.db_client) # Set up shutdown handlers cls.shutdown_lock = threading.Lock() @@ -672,7 +672,7 @@ def run_service(self): ) self.credentials_store = SupabaseIntegrationCredentialsStore( - self.supabase, redis.get_redis() + redis=redis.get_redis(), db=self.db_client ) self.executor = ProcessPoolExecutor( max_workers=self.pool_size, @@ -703,7 +703,7 @@ def cleanup(self): super().cleanup() - @thread_cached_property + @property def db_client(self) -> "DatabaseManager": return get_db_client() @@ -859,6 +859,7 @@ def _validate_node_input_credentials(self, graph: Graph, user_id: str): # ------- UTILITIES ------- # +@thread_cached def get_db_client() -> "DatabaseManager": from backend.executor import DatabaseManager diff --git a/autogpt_platform/backend/backend/integrations/creds_manager.py b/autogpt_platform/backend/backend/integrations/creds_manager.py index 6fcee8eecfee..3bfde70e2817 100644 --- a/autogpt_platform/backend/backend/integrations/creds_manager.py +++ b/autogpt_platform/backend/backend/integrations/creds_manager.py @@ -10,11 +10,10 @@ from redis.lock import Lock as RedisLock from backend.data import redis +from backend.executor.database import DatabaseManager from backend.integrations.oauth import HANDLERS_BY_NAME, BaseOAuthHandler from backend.util.settings import Settings -from ..server.integrations.utils import get_supabase - logger = logging.getLogger(__name__) settings = Settings() @@ -51,10 +50,12 @@ class IntegrationCredentialsManager: cause so much latency that it's worth implementing. """ - def __init__(self): + def __init__(self, db_manager: DatabaseManager): redis_conn = redis.get_redis() self._locks = RedisKeyedMutex(redis_conn) - self.store = SupabaseIntegrationCredentialsStore(get_supabase(), redis_conn) + self.store = SupabaseIntegrationCredentialsStore( + redis=redis_conn, db=db_manager + ) def create(self, user_id: str, credentials: Credentials) -> None: return self.store.add_creds(user_id, credentials) @@ -131,7 +132,7 @@ def delete(self, user_id: str, credentials_id: str) -> None: def _acquire_lock(self, user_id: str, credentials_id: str, *args: str) -> RedisLock: key = ( - self.store.supabase.supabase_url, + self.store.db_manager, f"user:{user_id}", f"credentials:{credentials_id}", *args, diff --git a/autogpt_platform/backend/backend/server/integrations/router.py b/autogpt_platform/backend/backend/server/integrations/router.py index a44954b37157..440ce7921cbe 100644 --- a/autogpt_platform/backend/backend/server/integrations/router.py +++ b/autogpt_platform/backend/backend/server/integrations/router.py @@ -10,6 +10,7 @@ from fastapi import APIRouter, Body, Depends, HTTPException, Path, Query, Request from pydantic import BaseModel, Field, SecretStr +from backend.executor.manager import get_db_client from backend.integrations.creds_manager import IntegrationCredentialsManager from backend.integrations.oauth import HANDLERS_BY_NAME, BaseOAuthHandler from backend.util.settings import Settings @@ -19,7 +20,8 @@ logger = logging.getLogger(__name__) settings = Settings() router = APIRouter() -creds_manager = IntegrationCredentialsManager() + +creds_manager = IntegrationCredentialsManager(db_manager=get_db_client()) class LoginResponse(BaseModel): @@ -41,7 +43,7 @@ async def login( requested_scopes = scopes.split(",") if scopes else [] # Generate and store a secure random state token along with the scopes - state_token = await creds_manager.store.store_state_token( + state_token = creds_manager.store.store_state_token( user_id, provider, requested_scopes ) @@ -70,12 +72,12 @@ async def callback( handler = _get_provider_oauth_handler(request, provider) # Verify the state token - if not await creds_manager.store.verify_state_token(user_id, state_token, provider): + if not creds_manager.store.verify_state_token(user_id, state_token, provider): logger.warning(f"Invalid or expired state token for user {user_id}") raise HTTPException(status_code=400, detail="Invalid or expired state token") try: - scopes = await creds_manager.store.get_any_valid_scopes_from_state_token( + scopes = creds_manager.store.get_any_valid_scopes_from_state_token( user_id, state_token, provider ) logger.debug(f"Retrieved scopes from state token: {scopes}") diff --git a/autogpt_platform/backend/backend/server/rest_api.py b/autogpt_platform/backend/backend/server/rest_api.py index a741f6ba353d..edeb025803c6 100644 --- a/autogpt_platform/backend/backend/server/rest_api.py +++ b/autogpt_platform/backend/backend/server/rest_api.py @@ -19,6 +19,7 @@ from backend.data.credit import get_block_costs, get_user_credit_model from backend.data.user import get_or_create_user from backend.executor import ExecutionManager, ExecutionScheduler +from backend.executor.manager import get_db_client from backend.integrations.creds_manager import IntegrationCredentialsManager from backend.server.model import CreateGraph, SetGraphActiveVersion from backend.util.cache import thread_cached_property @@ -97,7 +98,7 @@ def run_service(self): tags=["integrations"], dependencies=[Depends(auth_middleware)], ) - self.integration_creds_manager = IntegrationCredentialsManager() + self.integration_creds_manager = IntegrationCredentialsManager(get_db_client()) api_router.include_router( backend.server.routers.analytics.router, diff --git a/autogpt_platform/backend/backend/util/cache.py b/autogpt_platform/backend/backend/util/cache.py index 30048e8781af..b4506dda47b8 100644 --- a/autogpt_platform/backend/backend/util/cache.py +++ b/autogpt_platform/backend/backend/util/cache.py @@ -1,21 +1,27 @@ import threading from functools import wraps -from typing import Callable, TypeVar +from typing import Callable, ParamSpec, TypeVar T = TypeVar("T") +P = ParamSpec("P") R = TypeVar("R") -def thread_cached_property(func: Callable[[T], R]) -> property: - local_cache = threading.local() +def thread_cached(func: Callable[P, R]) -> Callable[P, R]: + thread_local = threading.local() @wraps(func) - def wrapper(self: T) -> R: - if not hasattr(local_cache, "cache"): - local_cache.cache = {} - key = id(self) - if key not in local_cache.cache: - local_cache.cache[key] = func(self) - return local_cache.cache[key] + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: + cache = getattr(thread_local, "cache", None) + if cache is None: + cache = thread_local.cache = {} + key = (args, tuple(sorted(kwargs.items()))) + if key not in cache: + cache[key] = func(*args, **kwargs) + return cache[key] + + return wrapper - return property(wrapper) + +def thread_cached_property(func: Callable[[T], R]) -> property: + return property(thread_cached(func)) diff --git a/autogpt_platform/backend/migrations/20241007175111_move_oauth_creds_to_user_obj/migration.sql b/autogpt_platform/backend/migrations/20241007175111_move_oauth_creds_to_user_obj/migration.sql new file mode 100644 index 000000000000..b3886efa030a --- /dev/null +++ b/autogpt_platform/backend/migrations/20241007175111_move_oauth_creds_to_user_obj/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "User" ADD COLUMN "metadata" JSONB; diff --git a/autogpt_platform/backend/migrations/20241007175112_add_oauth_creds_user_trigger/migration.sql b/autogpt_platform/backend/migrations/20241007175112_add_oauth_creds_user_trigger/migration.sql new file mode 100644 index 000000000000..aa577c90e938 --- /dev/null +++ b/autogpt_platform/backend/migrations/20241007175112_add_oauth_creds_user_trigger/migration.sql @@ -0,0 +1,27 @@ +--CreateFunction +CREATE OR REPLACE FUNCTION add_user_to_platform() RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO platform."User" (id, email, "updatedAt") + VALUES (NEW.id, NEW.email, now()); + RETURN NEW; +END; +$$ LANGUAGE plpgsql SECURITY DEFINER; + +DO $$ +BEGIN + -- Check if the auth schema and users table exist + IF EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'auth' + AND table_name = 'users' + ) THEN + -- Drop the trigger if it exists + DROP TRIGGER IF EXISTS user_added_to_platform ON auth.users; + + -- Create the trigger + CREATE TRIGGER user_added_to_platform + AFTER INSERT ON auth.users + FOR EACH ROW EXECUTE FUNCTION add_user_to_platform(); + END IF; +END $$; diff --git a/autogpt_platform/backend/schema.prisma b/autogpt_platform/backend/schema.prisma index 3fab8dc2593d..b316e226d202 100644 --- a/autogpt_platform/backend/schema.prisma +++ b/autogpt_platform/backend/schema.prisma @@ -17,6 +17,7 @@ model User { name String? createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + metadata Json? // Relations AgentGraphs AgentGraph[] diff --git a/autogpt_platform/docker-compose.platform.yml b/autogpt_platform/docker-compose.platform.yml index 020f32201fdc..ba396c76eac3 100644 --- a/autogpt_platform/docker-compose.platform.yml +++ b/autogpt_platform/docker-compose.platform.yml @@ -8,7 +8,7 @@ services: develop: watch: - path: ./ - target: autogpt_platform/backend/migrate + target: autogpt_platform/backend/migrations action: rebuild depends_on: db: diff --git a/autogpt_platform/docker-compose.yml b/autogpt_platform/docker-compose.yml index dcde6567f178..be6f1f49ede5 100644 --- a/autogpt_platform/docker-compose.yml +++ b/autogpt_platform/docker-compose.yml @@ -96,6 +96,36 @@ services: file: ./supabase/docker/docker-compose.yml service: rest + realtime: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: realtime + + storage: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: storage + + imgproxy: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: imgproxy + + meta: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: meta + + functions: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: functions + analytics: <<: *supabase-services extends: diff --git a/autogpt_platform/frontend/src/app/marketplace/submit/page.tsx b/autogpt_platform/frontend/src/app/marketplace/submit/page.tsx index 63ee9cfd2f0b..24863130a827 100644 --- a/autogpt_platform/frontend/src/app/marketplace/submit/page.tsx +++ b/autogpt_platform/frontend/src/app/marketplace/submit/page.tsx @@ -144,7 +144,7 @@ const SubmitPage: React.FC = () => { setSubmitError(null); if (!data.agreeToTerms) { - throw new Error("You must agree to the terms of service"); + throw new Error("You must agree to the terms of use"); } try { @@ -404,7 +404,7 @@ const SubmitPage: React.FC = () => { (
{ className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70" > I agree to the{" "} - - terms of service + + terms of use
diff --git a/autogpt_platform/infra/helm/autogpt-builder/values.dev.yaml b/autogpt_platform/infra/helm/autogpt-builder/values.dev.yaml index 1821acc24a39..128ea3ee44a5 100644 --- a/autogpt_platform/infra/helm/autogpt-builder/values.dev.yaml +++ b/autogpt_platform/infra/helm/autogpt-builder/values.dev.yaml @@ -1,9 +1,9 @@ # dev values, overwrite base values as needed. image: - repository: us-east1-docker.pkg.dev/agpt-dev/agpt-builder-dev/agpt-builder-dev + repository: us-east1-docker.pkg.dev/agpt-dev/agpt-frontend-dev/agpt-frontend-dev pullPolicy: Always - tag: "fe3d2a9" + tag: "latest" serviceAccount: annotations: diff --git a/autogpt_platform/infra/helm/autogpt-server/values.dev.yaml b/autogpt_platform/infra/helm/autogpt-server/values.dev.yaml index 6e86991623ec..b7488e0fd078 100644 --- a/autogpt_platform/infra/helm/autogpt-server/values.dev.yaml +++ b/autogpt_platform/infra/helm/autogpt-server/values.dev.yaml @@ -1,7 +1,7 @@ # dev values, overwrite base values as needed. image: - repository: us-east1-docker.pkg.dev/agpt-dev/agpt-server-dev/agpt-server-dev + repository: us-east1-docker.pkg.dev/agpt-dev/agpt-backend-dev/agpt-backend-dev pullPolicy: Always tag: "latest" diff --git a/autogpt_platform/infra/helm/autogpt-websocket-server/values.dev.yaml b/autogpt_platform/infra/helm/autogpt-websocket-server/values.dev.yaml index a977f9bece0c..dd26b32f4030 100644 --- a/autogpt_platform/infra/helm/autogpt-websocket-server/values.dev.yaml +++ b/autogpt_platform/infra/helm/autogpt-websocket-server/values.dev.yaml @@ -1,7 +1,7 @@ replicaCount: 1 # not scaling websocket server for now image: - repository: us-east1-docker.pkg.dev/agpt-dev/agpt-server-dev/agpt-server-dev + repository: us-east1-docker.pkg.dev/agpt-dev/agpt-backend-dev/agpt-backend-dev tag: latest pullPolicy: Always diff --git a/autogpt_platform/infra/terraform/environments/dev.tfvars b/autogpt_platform/infra/terraform/environments/dev.tfvars index 1f72c6d907a0..c193e8399a9a 100644 --- a/autogpt_platform/infra/terraform/environments/dev.tfvars +++ b/autogpt_platform/infra/terraform/environments/dev.tfvars @@ -28,6 +28,10 @@ service_accounts = { "dev-agpt-market-sa" = { display_name = "AutoGPT Dev Market Server Account" description = "Service account for agpt dev market server" + }, + "dev-github-actions-sa" = { + display_name = "GitHub Actions Dev Service Account" + description = "Service account for GitHub Actions deployments to dev" } } @@ -51,6 +55,11 @@ workload_identity_bindings = { service_account_name = "dev-agpt-market-sa" namespace = "dev-agpt" ksa_name = "dev-agpt-market-sa" + }, + "dev-github-actions-workload-identity" = { + service_account_name = "dev-github-actions-sa" + namespace = "dev-agpt" + ksa_name = "dev-github-actions-sa" } } @@ -59,7 +68,8 @@ role_bindings = { "serviceAccount:dev-agpt-server-sa@agpt-dev.iam.gserviceaccount.com", "serviceAccount:dev-agpt-builder-sa@agpt-dev.iam.gserviceaccount.com", "serviceAccount:dev-agpt-ws-server-sa@agpt-dev.iam.gserviceaccount.com", - "serviceAccount:dev-agpt-market-sa@agpt-dev.iam.gserviceaccount.com" + "serviceAccount:dev-agpt-market-sa@agpt-dev.iam.gserviceaccount.com", + "serviceAccount:dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com" ], "roles/cloudsql.client" = [ "serviceAccount:dev-agpt-server-sa@agpt-dev.iam.gserviceaccount.com", @@ -80,7 +90,8 @@ role_bindings = { "serviceAccount:dev-agpt-server-sa@agpt-dev.iam.gserviceaccount.com", "serviceAccount:dev-agpt-builder-sa@agpt-dev.iam.gserviceaccount.com", "serviceAccount:dev-agpt-ws-server-sa@agpt-dev.iam.gserviceaccount.com", - "serviceAccount:dev-agpt-market-sa@agpt-dev.iam.gserviceaccount.com" + "serviceAccount:dev-agpt-market-sa@agpt-dev.iam.gserviceaccount.com", + "serviceAccount:dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com" ] "roles/compute.networkUser" = [ "serviceAccount:dev-agpt-server-sa@agpt-dev.iam.gserviceaccount.com", @@ -93,6 +104,16 @@ role_bindings = { "serviceAccount:dev-agpt-builder-sa@agpt-dev.iam.gserviceaccount.com", "serviceAccount:dev-agpt-ws-server-sa@agpt-dev.iam.gserviceaccount.com", "serviceAccount:dev-agpt-market-sa@agpt-dev.iam.gserviceaccount.com" + ], + "roles/artifactregistry.writer" = [ + "serviceAccount:dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com" + ], + "roles/container.viewer" = [ + "serviceAccount:dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com" + ], + "roles/iam.serviceAccountTokenCreator" = [ + "principalSet://iam.googleapis.com/projects/638488734936/locations/global/workloadIdentityPools/dev-pool/*", + "serviceAccount:dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com" ] } @@ -101,4 +122,25 @@ services_ip_cidr_range = "10.2.0.0/20" public_bucket_names = ["website-artifacts"] standard_bucket_names = [] -bucket_admins = ["gcp-devops-agpt@agpt.co", "gcp-developers@agpt.co"] \ No newline at end of file +bucket_admins = ["gcp-devops-agpt@agpt.co", "gcp-developers@agpt.co"] + +workload_identity_pools = { + "dev-pool" = { + display_name = "Development Identity Pool" + providers = { + "github" = { + issuer_uri = "https://token.actions.githubusercontent.com" + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.repository" = "assertion.repository" + "attribute.repository_owner" = "assertion.repository_owner" + } + } + } + service_accounts = { + "dev-github-actions-sa" = [ + "Significant-Gravitas/AutoGPT" + ] + } + } +} \ No newline at end of file diff --git a/autogpt_platform/infra/terraform/main.tf b/autogpt_platform/infra/terraform/main.tf index 31049ed40cb9..047ebd59c661 100644 --- a/autogpt_platform/infra/terraform/main.tf +++ b/autogpt_platform/infra/terraform/main.tf @@ -61,6 +61,7 @@ module "iam" { service_accounts = var.service_accounts workload_identity_bindings = var.workload_identity_bindings role_bindings = var.role_bindings + workload_identity_pools = var.workload_identity_pools } module "storage" { diff --git a/autogpt_platform/infra/terraform/modules/iam/main.tf b/autogpt_platform/infra/terraform/modules/iam/main.tf index 3a07d6926456..f632f4c42b43 100644 --- a/autogpt_platform/infra/terraform/modules/iam/main.tf +++ b/autogpt_platform/infra/terraform/modules/iam/main.tf @@ -23,4 +23,31 @@ resource "google_project_iam_binding" "role_bindings" { role = each.key members = each.value +} + +resource "google_iam_workload_identity_pool" "pools" { + for_each = var.workload_identity_pools + workload_identity_pool_id = each.key + display_name = each.value.display_name +} + +resource "google_iam_workload_identity_pool_provider" "providers" { + for_each = merge([ + for pool_id, pool in var.workload_identity_pools : { + for provider_id, provider in pool.providers : + "${pool_id}/${provider_id}" => merge(provider, { + pool_id = pool_id + }) + } + ]...) + + workload_identity_pool_id = split("/", each.key)[0] + workload_identity_pool_provider_id = split("/", each.key)[1] + + attribute_mapping = each.value.attribute_mapping + oidc { + issuer_uri = each.value.issuer_uri + allowed_audiences = each.value.allowed_audiences + } + attribute_condition = "assertion.repository_owner==\"Significant-Gravitas\"" } \ No newline at end of file diff --git a/autogpt_platform/infra/terraform/modules/iam/outputs.tf b/autogpt_platform/infra/terraform/modules/iam/outputs.tf index b503414873a9..19354364bebb 100644 --- a/autogpt_platform/infra/terraform/modules/iam/outputs.tf +++ b/autogpt_platform/infra/terraform/modules/iam/outputs.tf @@ -1,4 +1,14 @@ output "service_account_emails" { description = "The emails of the created service accounts" value = { for k, v in google_service_account.service_accounts : k => v.email } -} \ No newline at end of file +} + +output "workload_identity_pools" { + value = google_iam_workload_identity_pool.pools +} + +output "workload_identity_providers" { + value = { + for k, v in google_iam_workload_identity_pool_provider.providers : k => v.name + } +} diff --git a/autogpt_platform/infra/terraform/modules/iam/variables.tf b/autogpt_platform/infra/terraform/modules/iam/variables.tf index 61637a718783..c9563ea0c7b8 100644 --- a/autogpt_platform/infra/terraform/modules/iam/variables.tf +++ b/autogpt_platform/infra/terraform/modules/iam/variables.tf @@ -26,4 +26,17 @@ variable "role_bindings" { description = "Map of roles to list of members" type = map(list(string)) default = {} +} + +variable "workload_identity_pools" { + type = map(object({ + display_name = string + providers = map(object({ + issuer_uri = string + attribute_mapping = map(string) + allowed_audiences = optional(list(string)) + })) + service_accounts = map(list(string)) # Map of SA to list of allowed principals + })) + default = {} } \ No newline at end of file diff --git a/autogpt_platform/infra/terraform/variables.tf b/autogpt_platform/infra/terraform/variables.tf index 1afd9509fcbc..3b4eb92dff9c 100644 --- a/autogpt_platform/infra/terraform/variables.tf +++ b/autogpt_platform/infra/terraform/variables.tf @@ -130,3 +130,19 @@ variable "bucket_admins" { default = ["gcp-devops-agpt@agpt.co", "gcp-developers@agpt.co"] } +variable "workload_identity_pools" { + type = map(object({ + display_name = string + providers = map(object({ + issuer_uri = string + attribute_mapping = map(string) + allowed_audiences = optional(list(string)) + })) + service_accounts = map(list(string)) + })) + default = {} + description = "Configuration for workload identity pools and their providers" +} + + +