From 75f9b072a6432165a13ead5b4102b6245bddf6f8 Mon Sep 17 00:00:00 2001 From: Nicholas Tindle Date: Fri, 29 Nov 2024 05:48:04 -0600 Subject: [PATCH] refactor(backend): Rename & move `IntegrationCredentialsStore` to backend (#8648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move `autogpt_libs.supabase_integration_credentials_store` into `backend` - `.store` -> `backend.integrations.credentials_store` - `.types` -> added to `backend.data.model` - Rename `SupabaseIntegrationCredentialsStore` to `IntegrationCredentialsStore` We wanted to get a few security things in quickly in #8403 and had to make some compromises to do so. This picks those up and fixes them. - Resolves #8540 ### Checklist 📋 #### For code changes: - [x] I have clearly listed my changes in the PR description - [x] I have made a test plan - [x] I have tested my changes according to the test plan: --------- Co-authored-by: Reinier van der Leer Co-authored-by: Aarushi <50577581+aarushik93@users.noreply.github.com> --- .../__init__.py | 9 -- .../types.py | 75 ---------------- .../blocks/ai_image_generator_block.py | 8 +- .../backend/blocks/ai_music_generator.py | 8 +- .../blocks/ai_shortform_video_block.py | 8 +- .../backend/backend/blocks/discord.py | 8 +- .../backend/backend/blocks/github/_auth.py | 9 +- .../backend/backend/blocks/google/_auth.py | 3 +- .../backend/backend/blocks/google_maps.py | 8 +- .../backend/backend/blocks/hubspot/_auth.py | 3 +- .../backend/backend/blocks/ideogram.py | 8 +- .../backend/backend/blocks/jina/_auth.py | 3 +- .../backend/backend/blocks/llm.py | 8 +- .../backend/backend/blocks/medium.py | 2 +- .../backend/backend/blocks/pinecone.py | 8 +- .../backend/blocks/replicate_flux_advanced.py | 8 +- .../backend/backend/blocks/search.py | 8 +- .../backend/backend/blocks/talking_head.py | 8 +- .../backend/blocks/text_to_speech_block.py | 8 +- autogpt_platform/backend/backend/cli.py | 10 +++ .../backend/backend/data/__init__.py | 0 .../backend/backend/data/block.py | 8 +- .../backend/backend/data/block_cost_config.py | 25 +++--- .../backend/backend/data/graph.py | 7 +- .../backend/backend/data/model.py | 87 ++++++++++++++++++- autogpt_platform/backend/backend/data/user.py | 6 +- .../backend/backend/executor/manager.py | 8 +- .../integrations/credentials_store.py} | 14 +-- .../backend/integrations/creds_manager.py | 8 +- .../backend/integrations/oauth/base.py | 2 +- .../backend/integrations/oauth/github.py | 3 +- .../backend/integrations/oauth/google.py | 3 +- .../backend/integrations/oauth/notion.py | 3 +- .../backend/integrations/webhooks/base.py | 2 +- .../backend/integrations/webhooks/github.py | 2 +- .../webhooks/graph_lifecycle_hooks.py | 3 +- .../backend/server/integrations/router.py | 12 +-- .../backend/backend/server/routers/v1.py | 2 +- .../backend/test/data/test_credit.py | 2 +- .../src/lib/autogpt-server-api/types.ts | 6 +- docs/content/platform/getting-started.md | 18 ++++ docs/content/platform/new_blocks.md | 2 +- 42 files changed, 246 insertions(+), 187 deletions(-) delete mode 100644 autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/__init__.py delete mode 100644 autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/types.py delete mode 100644 autogpt_platform/backend/backend/data/__init__.py rename autogpt_platform/{autogpt_libs/autogpt_libs/supabase_integration_credentials_store/store.py => backend/backend/integrations/credentials_store.py} (98%) diff --git a/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/__init__.py b/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/__init__.py deleted file mode 100644 index bae73dfbf269..000000000000 --- a/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from .store import SupabaseIntegrationCredentialsStore -from .types import APIKeyCredentials, Credentials, OAuth2Credentials - -__all__ = [ - "SupabaseIntegrationCredentialsStore", - "Credentials", - "APIKeyCredentials", - "OAuth2Credentials", -] diff --git a/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/types.py b/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/types.py deleted file mode 100644 index 384dfdc79729..000000000000 --- a/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/types.py +++ /dev/null @@ -1,75 +0,0 @@ -from typing import Annotated, Any, Literal, Optional, TypedDict -from uuid import uuid4 - -from pydantic import BaseModel, Field, SecretStr, field_serializer - - -class _BaseCredentials(BaseModel): - id: str = Field(default_factory=lambda: str(uuid4())) - provider: str - title: Optional[str] - - @field_serializer("*") - def dump_secret_strings(value: Any, _info): - if isinstance(value, SecretStr): - return value.get_secret_value() - return value - - -class OAuth2Credentials(_BaseCredentials): - type: Literal["oauth2"] = "oauth2" - username: Optional[str] - """Username of the third-party service user that these credentials belong to""" - access_token: SecretStr - access_token_expires_at: Optional[int] - """Unix timestamp (seconds) indicating when the access token expires (if at all)""" - refresh_token: Optional[SecretStr] - refresh_token_expires_at: Optional[int] - """Unix timestamp (seconds) indicating when the refresh token expires (if at all)""" - scopes: list[str] - metadata: dict[str, Any] = Field(default_factory=dict) - - def bearer(self) -> str: - return f"Bearer {self.access_token.get_secret_value()}" - - -class APIKeyCredentials(_BaseCredentials): - type: Literal["api_key"] = "api_key" - api_key: SecretStr - expires_at: Optional[int] - """Unix timestamp (seconds) indicating when the API key expires (if at all)""" - - def bearer(self) -> str: - return f"Bearer {self.api_key.get_secret_value()}" - - -Credentials = Annotated[ - OAuth2Credentials | APIKeyCredentials, - Field(discriminator="type"), -] - - -CredentialsType = Literal["api_key", "oauth2"] - - -class OAuthState(BaseModel): - token: str - provider: str - expires_at: int - scopes: list[str] - """Unix timestamp (seconds) indicating when this OAuth state expires""" - - -class UserMetadata(BaseModel): - integration_credentials: list[Credentials] = Field(default_factory=list) - integration_oauth_states: list[OAuthState] = Field(default_factory=list) - - -class UserMetadataRaw(TypedDict, total=False): - integration_credentials: list[dict] - integration_oauth_states: list[dict] - - -class UserIntegrations(BaseModel): - credentials: list[Credentials] = Field(default_factory=list) - oauth_states: list[OAuthState] = Field(default_factory=list) diff --git a/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py b/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py index f2441708ce31..ee94f016aa55 100644 --- a/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py +++ b/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py @@ -2,12 +2,16 @@ from typing import Literal import replicate -from autogpt_libs.supabase_integration_credentials_store.types import APIKeyCredentials from pydantic import SecretStr from replicate.helpers import FileOutput from backend.data.block import Block, BlockCategory, BlockSchema -from backend.data.model import CredentialsField, CredentialsMetaInput, SchemaField +from backend.data.model import ( + APIKeyCredentials, + CredentialsField, + CredentialsMetaInput, + SchemaField, +) class ImageSize(str, Enum): diff --git a/autogpt_platform/backend/backend/blocks/ai_music_generator.py b/autogpt_platform/backend/backend/blocks/ai_music_generator.py index 6b2c086a0a80..f70d43ce370e 100644 --- a/autogpt_platform/backend/backend/blocks/ai_music_generator.py +++ b/autogpt_platform/backend/backend/blocks/ai_music_generator.py @@ -4,11 +4,15 @@ from typing import Literal import replicate -from autogpt_libs.supabase_integration_credentials_store.types import APIKeyCredentials from pydantic import SecretStr from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema -from backend.data.model import CredentialsField, CredentialsMetaInput, SchemaField +from backend.data.model import ( + APIKeyCredentials, + CredentialsField, + CredentialsMetaInput, + SchemaField, +) logger = logging.getLogger(__name__) diff --git a/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py b/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py index e70f8fd40e7d..08023b877118 100644 --- a/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py +++ b/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py @@ -3,11 +3,15 @@ from enum import Enum from typing import Literal -from autogpt_libs.supabase_integration_credentials_store.types import APIKeyCredentials from pydantic import SecretStr from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema -from backend.data.model import CredentialsField, CredentialsMetaInput, SchemaField +from backend.data.model import ( + APIKeyCredentials, + CredentialsField, + CredentialsMetaInput, + SchemaField, +) from backend.util.request import requests TEST_CREDENTIALS = APIKeyCredentials( diff --git a/autogpt_platform/backend/backend/blocks/discord.py b/autogpt_platform/backend/backend/blocks/discord.py index 69d3c3bc744c..c638e402508b 100644 --- a/autogpt_platform/backend/backend/blocks/discord.py +++ b/autogpt_platform/backend/backend/blocks/discord.py @@ -3,11 +3,15 @@ import aiohttp import discord -from autogpt_libs.supabase_integration_credentials_store.types import APIKeyCredentials from pydantic import SecretStr from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema -from backend.data.model import CredentialsField, CredentialsMetaInput, SchemaField +from backend.data.model import ( + APIKeyCredentials, + CredentialsField, + CredentialsMetaInput, + SchemaField, +) DiscordCredentials = CredentialsMetaInput[Literal["discord"], Literal["api_key"]] diff --git a/autogpt_platform/backend/backend/blocks/github/_auth.py b/autogpt_platform/backend/backend/blocks/github/_auth.py index 4ea31e98be9f..72aa8f648015 100644 --- a/autogpt_platform/backend/backend/blocks/github/_auth.py +++ b/autogpt_platform/backend/backend/blocks/github/_auth.py @@ -1,12 +1,13 @@ from typing import Literal -from autogpt_libs.supabase_integration_credentials_store.types import ( +from pydantic import SecretStr + +from backend.data.model import ( APIKeyCredentials, + CredentialsField, + CredentialsMetaInput, OAuth2Credentials, ) -from pydantic import SecretStr - -from backend.data.model import CredentialsField, CredentialsMetaInput from backend.util.settings import Secrets secrets = Secrets() diff --git a/autogpt_platform/backend/backend/blocks/google/_auth.py b/autogpt_platform/backend/backend/blocks/google/_auth.py index 742fcb36fa17..ccae2e462244 100644 --- a/autogpt_platform/backend/backend/blocks/google/_auth.py +++ b/autogpt_platform/backend/backend/blocks/google/_auth.py @@ -1,9 +1,8 @@ from typing import Literal -from autogpt_libs.supabase_integration_credentials_store.types import OAuth2Credentials from pydantic import SecretStr -from backend.data.model import CredentialsField, CredentialsMetaInput +from backend.data.model import CredentialsField, CredentialsMetaInput, OAuth2Credentials from backend.util.settings import Secrets # --8<-- [start:GoogleOAuthIsConfigured] diff --git a/autogpt_platform/backend/backend/blocks/google_maps.py b/autogpt_platform/backend/backend/blocks/google_maps.py index 97d91c83705e..d211fe8ff3f0 100644 --- a/autogpt_platform/backend/backend/blocks/google_maps.py +++ b/autogpt_platform/backend/backend/blocks/google_maps.py @@ -1,11 +1,15 @@ from typing import Literal import googlemaps -from autogpt_libs.supabase_integration_credentials_store.types import APIKeyCredentials from pydantic import BaseModel, SecretStr from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema -from backend.data.model import CredentialsField, CredentialsMetaInput, SchemaField +from backend.data.model import ( + APIKeyCredentials, + CredentialsField, + CredentialsMetaInput, + SchemaField, +) TEST_CREDENTIALS = APIKeyCredentials( id="01234567-89ab-cdef-0123-456789abcdef", diff --git a/autogpt_platform/backend/backend/blocks/hubspot/_auth.py b/autogpt_platform/backend/backend/blocks/hubspot/_auth.py index 3fa5f6494561..c32af8c38c01 100644 --- a/autogpt_platform/backend/backend/blocks/hubspot/_auth.py +++ b/autogpt_platform/backend/backend/blocks/hubspot/_auth.py @@ -1,9 +1,8 @@ from typing import Literal -from autogpt_libs.supabase_integration_credentials_store.types import APIKeyCredentials from pydantic import SecretStr -from backend.data.model import CredentialsField, CredentialsMetaInput +from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput HubSpotCredentials = APIKeyCredentials HubSpotCredentialsInput = CredentialsMetaInput[ diff --git a/autogpt_platform/backend/backend/blocks/ideogram.py b/autogpt_platform/backend/backend/blocks/ideogram.py index 8c9e4b6f5933..b6a21e7acec2 100644 --- a/autogpt_platform/backend/backend/blocks/ideogram.py +++ b/autogpt_platform/backend/backend/blocks/ideogram.py @@ -1,12 +1,16 @@ from enum import Enum from typing import Any, Dict, Literal, Optional -from autogpt_libs.supabase_integration_credentials_store.types import APIKeyCredentials from pydantic import SecretStr from requests.exceptions import RequestException from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema -from backend.data.model import CredentialsField, CredentialsMetaInput, SchemaField +from backend.data.model import ( + APIKeyCredentials, + CredentialsField, + CredentialsMetaInput, + SchemaField, +) from backend.util.request import requests TEST_CREDENTIALS = APIKeyCredentials( diff --git a/autogpt_platform/backend/backend/blocks/jina/_auth.py b/autogpt_platform/backend/backend/blocks/jina/_auth.py index c8d9f6b05a77..2bffeecce754 100644 --- a/autogpt_platform/backend/backend/blocks/jina/_auth.py +++ b/autogpt_platform/backend/backend/blocks/jina/_auth.py @@ -1,9 +1,8 @@ from typing import Literal -from autogpt_libs.supabase_integration_credentials_store.types import APIKeyCredentials from pydantic import SecretStr -from backend.data.model import CredentialsField, CredentialsMetaInput +from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput JinaCredentials = APIKeyCredentials JinaCredentialsInput = CredentialsMetaInput[ diff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.py index a30ed1576ccd..50a9bfb3c626 100644 --- a/autogpt_platform/backend/backend/blocks/llm.py +++ b/autogpt_platform/backend/backend/blocks/llm.py @@ -5,7 +5,6 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, List, Literal, NamedTuple -from autogpt_libs.supabase_integration_credentials_store.types import APIKeyCredentials from pydantic import SecretStr if TYPE_CHECKING: @@ -17,7 +16,12 @@ from groq import Groq from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema -from backend.data.model import CredentialsField, CredentialsMetaInput, SchemaField +from backend.data.model import ( + APIKeyCredentials, + CredentialsField, + CredentialsMetaInput, + SchemaField, +) from backend.util import json from backend.util.settings import BehaveAs, Settings diff --git a/autogpt_platform/backend/backend/blocks/medium.py b/autogpt_platform/backend/backend/blocks/medium.py index 60abb4659808..da8c367c7860 100644 --- a/autogpt_platform/backend/backend/blocks/medium.py +++ b/autogpt_platform/backend/backend/blocks/medium.py @@ -1,11 +1,11 @@ from enum import Enum from typing import List, Literal -from autogpt_libs.supabase_integration_credentials_store.types import APIKeyCredentials from pydantic import SecretStr from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema from backend.data.model import ( + APIKeyCredentials, BlockSecret, CredentialsField, CredentialsMetaInput, diff --git a/autogpt_platform/backend/backend/blocks/pinecone.py b/autogpt_platform/backend/backend/blocks/pinecone.py index 5ef8e639920f..a62c1fa77739 100644 --- a/autogpt_platform/backend/backend/blocks/pinecone.py +++ b/autogpt_platform/backend/backend/blocks/pinecone.py @@ -1,11 +1,15 @@ import uuid from typing import Any, Literal -from autogpt_libs.supabase_integration_credentials_store import APIKeyCredentials from pinecone import Pinecone, ServerlessSpec from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema -from backend.data.model import CredentialsField, CredentialsMetaInput, SchemaField +from backend.data.model import ( + APIKeyCredentials, + CredentialsField, + CredentialsMetaInput, + SchemaField, +) PineconeCredentials = APIKeyCredentials PineconeCredentialsInput = CredentialsMetaInput[ diff --git a/autogpt_platform/backend/backend/blocks/replicate_flux_advanced.py b/autogpt_platform/backend/backend/blocks/replicate_flux_advanced.py index 9200084c470d..b346c87a38a0 100644 --- a/autogpt_platform/backend/backend/blocks/replicate_flux_advanced.py +++ b/autogpt_platform/backend/backend/blocks/replicate_flux_advanced.py @@ -3,12 +3,16 @@ from typing import Literal import replicate -from autogpt_libs.supabase_integration_credentials_store.types import APIKeyCredentials from pydantic import SecretStr from replicate.helpers import FileOutput from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema -from backend.data.model import CredentialsField, CredentialsMetaInput, SchemaField +from backend.data.model import ( + APIKeyCredentials, + CredentialsField, + CredentialsMetaInput, + SchemaField, +) TEST_CREDENTIALS = APIKeyCredentials( id="01234567-89ab-cdef-0123-456789abcdef", diff --git a/autogpt_platform/backend/backend/blocks/search.py b/autogpt_platform/backend/backend/blocks/search.py index a73676269975..f89232703aa6 100644 --- a/autogpt_platform/backend/backend/blocks/search.py +++ b/autogpt_platform/backend/backend/blocks/search.py @@ -1,12 +1,16 @@ from typing import Literal from urllib.parse import quote -from autogpt_libs.supabase_integration_credentials_store.types import APIKeyCredentials from pydantic import SecretStr from backend.blocks.helpers.http import GetRequest from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema -from backend.data.model import CredentialsField, CredentialsMetaInput, SchemaField +from backend.data.model import ( + APIKeyCredentials, + CredentialsField, + CredentialsMetaInput, + SchemaField, +) class GetWikipediaSummaryBlock(Block, GetRequest): diff --git a/autogpt_platform/backend/backend/blocks/talking_head.py b/autogpt_platform/backend/backend/blocks/talking_head.py index 9f372b0abf9d..20aadcd214fe 100644 --- a/autogpt_platform/backend/backend/blocks/talking_head.py +++ b/autogpt_platform/backend/backend/blocks/talking_head.py @@ -1,11 +1,15 @@ import time from typing import Literal -from autogpt_libs.supabase_integration_credentials_store.types import APIKeyCredentials from pydantic import SecretStr from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema -from backend.data.model import CredentialsField, CredentialsMetaInput, SchemaField +from backend.data.model import ( + APIKeyCredentials, + CredentialsField, + CredentialsMetaInput, + SchemaField, +) from backend.util.request import requests TEST_CREDENTIALS = APIKeyCredentials( diff --git a/autogpt_platform/backend/backend/blocks/text_to_speech_block.py b/autogpt_platform/backend/backend/blocks/text_to_speech_block.py index 510d02817651..b92cc9fa4468 100644 --- a/autogpt_platform/backend/backend/blocks/text_to_speech_block.py +++ b/autogpt_platform/backend/backend/blocks/text_to_speech_block.py @@ -1,10 +1,14 @@ from typing import Any, Literal -from autogpt_libs.supabase_integration_credentials_store.types import APIKeyCredentials from pydantic import SecretStr from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema -from backend.data.model import CredentialsField, CredentialsMetaInput, SchemaField +from backend.data.model import ( + APIKeyCredentials, + CredentialsField, + CredentialsMetaInput, + SchemaField, +) from backend.util.request import requests TEST_CREDENTIALS = APIKeyCredentials( diff --git a/autogpt_platform/backend/backend/cli.py b/autogpt_platform/backend/backend/cli.py index 154c22207bb6..efaadd02b282 100755 --- a/autogpt_platform/backend/backend/cli.py +++ b/autogpt_platform/backend/backend/cli.py @@ -93,6 +93,16 @@ def stop(): print("Server Stopped") +@main.command() +def gen_encrypt_key(): + """ + Generate a new encryption key + """ + from cryptography.fernet import Fernet + + print(Fernet.generate_key().decode()) + + @click.group() def test(): """ diff --git a/autogpt_platform/backend/backend/data/__init__.py b/autogpt_platform/backend/backend/data/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/autogpt_platform/backend/backend/data/block.py b/autogpt_platform/backend/backend/data/block.py index 2ce8b75d1b83..e1035e84fe18 100644 --- a/autogpt_platform/backend/backend/data/block.py +++ b/autogpt_platform/backend/backend/data/block.py @@ -15,14 +15,18 @@ import jsonref import jsonschema -from autogpt_libs.supabase_integration_credentials_store.types import Credentials from prisma.models import AgentBlock from pydantic import BaseModel from backend.util import json from backend.util.settings import Config -from .model import CREDENTIALS_FIELD_NAME, ContributorDetails, CredentialsMetaInput +from .model import ( + CREDENTIALS_FIELD_NAME, + ContributorDetails, + Credentials, + CredentialsMetaInput, +) app_config = Config() diff --git a/autogpt_platform/backend/backend/data/block_cost_config.py b/autogpt_platform/backend/backend/data/block_cost_config.py index 42b43d95dceb..6b5a12409770 100644 --- a/autogpt_platform/backend/backend/data/block_cost_config.py +++ b/autogpt_platform/backend/backend/data/block_cost_config.py @@ -1,18 +1,5 @@ from typing import Type -from autogpt_libs.supabase_integration_credentials_store.store import ( - anthropic_credentials, - did_credentials, - groq_credentials, - ideogram_credentials, - jina_credentials, - open_router_credentials, - openai_credentials, - replicate_credentials, - revid_credentials, - unreal_credentials, -) - from backend.blocks.ai_music_generator import AIMusicGeneratorBlock from backend.blocks.ai_shortform_video_block import AIShortformVideoCreatorBlock from backend.blocks.ideogram import IdeogramModelBlock @@ -32,6 +19,18 @@ from backend.blocks.text_to_speech_block import UnrealTextToSpeechBlock from backend.data.block import Block from backend.data.cost import BlockCost, BlockCostType +from backend.integrations.credentials_store import ( + anthropic_credentials, + did_credentials, + groq_credentials, + ideogram_credentials, + jina_credentials, + open_router_credentials, + openai_credentials, + replicate_credentials, + revid_credentials, + unreal_credentials, +) # =============== Configure the cost for each LLM Model call =============== # diff --git a/autogpt_platform/backend/backend/data/graph.py b/autogpt_platform/backend/backend/data/graph.py index 1081d6b90e7c..dbe7b753b8c8 100644 --- a/autogpt_platform/backend/backend/data/graph.py +++ b/autogpt_platform/backend/backend/data/graph.py @@ -615,14 +615,11 @@ def make_graph_model(creatable_graph: Graph, user_id: str) -> GraphModel: async def fix_llm_provider_credentials(): """Fix node credentials with provider `llm`""" - from autogpt_libs.supabase_integration_credentials_store import ( - SupabaseIntegrationCredentialsStore, - ) + from backend.integrations.credentials_store import IntegrationCredentialsStore - from .redis import get_redis from .user import get_user_integrations - store = SupabaseIntegrationCredentialsStore(get_redis()) + store = IntegrationCredentialsStore() broken_nodes = await prisma.get_client().query_raw( """ diff --git a/autogpt_platform/backend/backend/data/model.py b/autogpt_platform/backend/backend/data/model.py index 9a988133eb59..f8b6781e0ce1 100644 --- a/autogpt_platform/backend/backend/data/model.py +++ b/autogpt_platform/backend/backend/data/model.py @@ -1,10 +1,20 @@ from __future__ import annotations import logging -from typing import Any, Callable, ClassVar, Generic, Optional, TypeVar +from typing import ( + Annotated, + Any, + Callable, + ClassVar, + Generic, + Literal, + Optional, + TypedDict, + TypeVar, +) +from uuid import uuid4 -from autogpt_libs.supabase_integration_credentials_store.types import CredentialsType -from pydantic import BaseModel, Field, GetCoreSchemaHandler +from pydantic import BaseModel, Field, GetCoreSchemaHandler, SecretStr, field_serializer from pydantic_core import ( CoreSchema, PydanticUndefined, @@ -139,6 +149,77 @@ def SchemaField( ) +class _BaseCredentials(BaseModel): + id: str = Field(default_factory=lambda: str(uuid4())) + provider: str + title: Optional[str] + + @field_serializer("*") + def dump_secret_strings(value: Any, _info): + if isinstance(value, SecretStr): + return value.get_secret_value() + return value + + +class OAuth2Credentials(_BaseCredentials): + type: Literal["oauth2"] = "oauth2" + username: Optional[str] + """Username of the third-party service user that these credentials belong to""" + access_token: SecretStr + access_token_expires_at: Optional[int] + """Unix timestamp (seconds) indicating when the access token expires (if at all)""" + refresh_token: Optional[SecretStr] + refresh_token_expires_at: Optional[int] + """Unix timestamp (seconds) indicating when the refresh token expires (if at all)""" + scopes: list[str] + metadata: dict[str, Any] = Field(default_factory=dict) + + def bearer(self) -> str: + return f"Bearer {self.access_token.get_secret_value()}" + + +class APIKeyCredentials(_BaseCredentials): + type: Literal["api_key"] = "api_key" + api_key: SecretStr + expires_at: Optional[int] + """Unix timestamp (seconds) indicating when the API key expires (if at all)""" + + def bearer(self) -> str: + return f"Bearer {self.api_key.get_secret_value()}" + + +Credentials = Annotated[ + OAuth2Credentials | APIKeyCredentials, + Field(discriminator="type"), +] + + +CredentialsType = Literal["api_key", "oauth2"] + + +class OAuthState(BaseModel): + token: str + provider: str + expires_at: int + """Unix timestamp (seconds) indicating when this OAuth state expires""" + scopes: list[str] + + +class UserMetadata(BaseModel): + integration_credentials: list[Credentials] = Field(default_factory=list) + integration_oauth_states: list[OAuthState] = Field(default_factory=list) + + +class UserMetadataRaw(TypedDict, total=False): + integration_credentials: list[dict] + integration_oauth_states: list[dict] + + +class UserIntegrations(BaseModel): + credentials: list[Credentials] = Field(default_factory=list) + oauth_states: list[OAuthState] = Field(default_factory=list) + + CP = TypeVar("CP", bound=str) CT = TypeVar("CT", bound=CredentialsType) diff --git a/autogpt_platform/backend/backend/data/user.py b/autogpt_platform/backend/backend/data/user.py index a9599bfddecd..8602d0f3b136 100644 --- a/autogpt_platform/backend/backend/data/user.py +++ b/autogpt_platform/backend/backend/data/user.py @@ -2,16 +2,12 @@ from typing import Optional, cast from autogpt_libs.auth.models import DEFAULT_USER_ID -from autogpt_libs.supabase_integration_credentials_store.types import ( - UserIntegrations, - UserMetadata, - UserMetadataRaw, -) from fastapi import HTTPException from prisma import Json from prisma.models import User from backend.data.db import prisma +from backend.data.model import UserIntegrations, UserMetadata, UserMetadataRaw from backend.util.encryption import JSONCryptor logger = logging.getLogger(__name__) diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py index 46cb554db8ab..c6db354049ca 100644 --- a/autogpt_platform/backend/backend/executor/manager.py +++ b/autogpt_platform/backend/backend/executor/manager.py @@ -725,13 +725,9 @@ def get_port(cls) -> int: return settings.config.execution_manager_port def run_service(self): - from autogpt_libs.supabase_integration_credentials_store import ( - SupabaseIntegrationCredentialsStore, - ) + from backend.integrations.credentials_store import IntegrationCredentialsStore - self.credentials_store = SupabaseIntegrationCredentialsStore( - redis=redis.get_redis() - ) + self.credentials_store = IntegrationCredentialsStore() self.executor = ProcessPoolExecutor( max_workers=self.pool_size, initializer=Executor.on_graph_executor_start, diff --git a/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/store.py b/autogpt_platform/backend/backend/integrations/credentials_store.py similarity index 98% rename from autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/store.py rename to autogpt_platform/backend/backend/integrations/credentials_store.py index c4167b3173bf..7d539b73c476 100644 --- a/autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/store.py +++ b/autogpt_platform/backend/backend/integrations/credentials_store.py @@ -6,20 +6,18 @@ if TYPE_CHECKING: from backend.executor.database import DatabaseManager - from redis import Redis from autogpt_libs.utils.cache import thread_cached from autogpt_libs.utils.synchronize import RedisKeyedMutex -from backend.util.settings import Settings - -from .types import ( +from backend.data.model import ( APIKeyCredentials, Credentials, OAuth2Credentials, OAuthState, UserIntegrations, ) +from backend.util.settings import Settings settings = Settings() @@ -109,9 +107,11 @@ ] -class SupabaseIntegrationCredentialsStore: - def __init__(self, redis: "Redis"): - self.locks = RedisKeyedMutex(redis) +class IntegrationCredentialsStore: + def __init__(self): + from backend.data.redis import get_redis + + self.locks = RedisKeyedMutex(get_redis()) @property @thread_cached diff --git a/autogpt_platform/backend/backend/integrations/creds_manager.py b/autogpt_platform/backend/backend/integrations/creds_manager.py index 9898afcc7906..7cbf8f4af7f1 100644 --- a/autogpt_platform/backend/backend/integrations/creds_manager.py +++ b/autogpt_platform/backend/backend/integrations/creds_manager.py @@ -2,14 +2,12 @@ from contextlib import contextmanager from datetime import datetime -from autogpt_libs.supabase_integration_credentials_store import ( - Credentials, - SupabaseIntegrationCredentialsStore, -) from autogpt_libs.utils.synchronize import RedisKeyedMutex from redis.lock import Lock as RedisLock from backend.data import redis +from backend.data.model import Credentials +from backend.integrations.credentials_store import IntegrationCredentialsStore from backend.integrations.oauth import HANDLERS_BY_NAME, BaseOAuthHandler from backend.util.exceptions import MissingConfigError from backend.util.settings import Settings @@ -53,7 +51,7 @@ class IntegrationCredentialsManager: def __init__(self): redis_conn = redis.get_redis() self._locks = RedisKeyedMutex(redis_conn) - self.store = SupabaseIntegrationCredentialsStore(redis=redis_conn) + self.store = IntegrationCredentialsStore() def create(self, user_id: str, credentials: Credentials) -> None: return self.store.add_creds(user_id, credentials) diff --git a/autogpt_platform/backend/backend/integrations/oauth/base.py b/autogpt_platform/backend/backend/integrations/oauth/base.py index a12200af6590..ad5433d734ec 100644 --- a/autogpt_platform/backend/backend/integrations/oauth/base.py +++ b/autogpt_platform/backend/backend/integrations/oauth/base.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod from typing import ClassVar -from autogpt_libs.supabase_integration_credentials_store import OAuth2Credentials +from backend.data.model import OAuth2Credentials logger = logging.getLogger(__name__) diff --git a/autogpt_platform/backend/backend/integrations/oauth/github.py b/autogpt_platform/backend/backend/integrations/oauth/github.py index 402dbe21a9f8..ed883b63205f 100644 --- a/autogpt_platform/backend/backend/integrations/oauth/github.py +++ b/autogpt_platform/backend/backend/integrations/oauth/github.py @@ -2,8 +2,7 @@ from typing import Optional from urllib.parse import urlencode -from autogpt_libs.supabase_integration_credentials_store import OAuth2Credentials - +from backend.data.model import OAuth2Credentials from backend.util.request import requests from .base import BaseOAuthHandler diff --git a/autogpt_platform/backend/backend/integrations/oauth/google.py b/autogpt_platform/backend/backend/integrations/oauth/google.py index 1f410b5e5920..13175b0b469f 100644 --- a/autogpt_platform/backend/backend/integrations/oauth/google.py +++ b/autogpt_platform/backend/backend/integrations/oauth/google.py @@ -1,6 +1,5 @@ import logging -from autogpt_libs.supabase_integration_credentials_store import OAuth2Credentials from google.auth.external_account_authorized_user import ( Credentials as ExternalAccountCredentials, ) @@ -9,6 +8,8 @@ from google_auth_oauthlib.flow import Flow from pydantic import SecretStr +from backend.data.model import OAuth2Credentials + from .base import BaseOAuthHandler logger = logging.getLogger(__name__) diff --git a/autogpt_platform/backend/backend/integrations/oauth/notion.py b/autogpt_platform/backend/backend/integrations/oauth/notion.py index 75f7a91eeef1..7f5458ae9a10 100644 --- a/autogpt_platform/backend/backend/integrations/oauth/notion.py +++ b/autogpt_platform/backend/backend/integrations/oauth/notion.py @@ -1,8 +1,7 @@ from base64 import b64encode from urllib.parse import urlencode -from autogpt_libs.supabase_integration_credentials_store import OAuth2Credentials - +from backend.data.model import OAuth2Credentials from backend.util.request import requests from .base import BaseOAuthHandler diff --git a/autogpt_platform/backend/backend/integrations/webhooks/base.py b/autogpt_platform/backend/backend/integrations/webhooks/base.py index b30f419a03e8..61a07ce20353 100644 --- a/autogpt_platform/backend/backend/integrations/webhooks/base.py +++ b/autogpt_platform/backend/backend/integrations/webhooks/base.py @@ -4,11 +4,11 @@ from typing import ClassVar, Generic, TypeVar from uuid import uuid4 -from autogpt_libs.supabase_integration_credentials_store import Credentials from fastapi import Request from strenum import StrEnum from backend.data import integrations +from backend.data.model import Credentials from backend.util.exceptions import MissingConfigError from backend.util.settings import Config diff --git a/autogpt_platform/backend/backend/integrations/webhooks/github.py b/autogpt_platform/backend/backend/integrations/webhooks/github.py index 25152caff438..2393437d209a 100644 --- a/autogpt_platform/backend/backend/integrations/webhooks/github.py +++ b/autogpt_platform/backend/backend/integrations/webhooks/github.py @@ -3,11 +3,11 @@ import logging import requests -from autogpt_libs.supabase_integration_credentials_store import Credentials from fastapi import HTTPException, Request from strenum import StrEnum from backend.data import integrations +from backend.data.model import Credentials from .base import BaseWebhooksManager diff --git a/autogpt_platform/backend/backend/integrations/webhooks/graph_lifecycle_hooks.py b/autogpt_platform/backend/backend/integrations/webhooks/graph_lifecycle_hooks.py index 1f6351d5fab2..363bf535c501 100644 --- a/autogpt_platform/backend/backend/integrations/webhooks/graph_lifecycle_hooks.py +++ b/autogpt_platform/backend/backend/integrations/webhooks/graph_lifecycle_hooks.py @@ -7,9 +7,8 @@ from backend.integrations.webhooks import WEBHOOK_MANAGERS_BY_NAME if TYPE_CHECKING: - from autogpt_libs.supabase_integration_credentials_store.types import Credentials - from backend.data.graph import GraphModel, NodeModel + from backend.data.model import Credentials from .base import BaseWebhooksManager diff --git a/autogpt_platform/backend/backend/server/integrations/router.py b/autogpt_platform/backend/backend/server/integrations/router.py index d18ba5c46e40..4220ac3c0311 100644 --- a/autogpt_platform/backend/backend/server/integrations/router.py +++ b/autogpt_platform/backend/backend/server/integrations/router.py @@ -1,12 +1,6 @@ import logging from typing import Annotated, Literal -from autogpt_libs.supabase_integration_credentials_store.types import ( - APIKeyCredentials, - Credentials, - CredentialsType, - OAuth2Credentials, -) from fastapi import APIRouter, Body, Depends, HTTPException, Path, Query, Request from pydantic import BaseModel, Field, SecretStr @@ -18,6 +12,12 @@ listen_for_webhook_event, publish_webhook_event, ) +from backend.data.model import ( + APIKeyCredentials, + Credentials, + CredentialsType, + OAuth2Credentials, +) from backend.executor.manager import ExecutionManager from backend.integrations.creds_manager import IntegrationCredentialsManager from backend.integrations.oauth import HANDLERS_BY_NAME, BaseOAuthHandler diff --git a/autogpt_platform/backend/backend/server/routers/v1.py b/autogpt_platform/backend/backend/server/routers/v1.py index 31a777a57293..c600c067c4db 100644 --- a/autogpt_platform/backend/backend/server/routers/v1.py +++ b/autogpt_platform/backend/backend/server/routers/v1.py @@ -48,7 +48,7 @@ from backend.util.settings import Settings if TYPE_CHECKING: - from autogpt_libs.supabase_integration_credentials_store.types import Credentials + from backend.data.model import Credentials @thread_cached diff --git a/autogpt_platform/backend/test/data/test_credit.py b/autogpt_platform/backend/test/data/test_credit.py index 77cef7735f5f..37236e3467ee 100644 --- a/autogpt_platform/backend/test/data/test_credit.py +++ b/autogpt_platform/backend/test/data/test_credit.py @@ -1,12 +1,12 @@ from datetime import datetime import pytest -from autogpt_libs.supabase_integration_credentials_store.store import openai_credentials from prisma.models import UserBlockCredit from backend.blocks.llm import AITextGeneratorBlock from backend.data.credit import UserCredit from backend.data.user import DEFAULT_USER_ID +from backend.integrations.credentials_store import openai_credentials from backend.util.test import SpinTestServer REFILL_VALUE = 1000 diff --git a/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts b/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts index fc42b49af91d..edeb87e55632 100644 --- a/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts +++ b/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts @@ -289,7 +289,7 @@ export type CredentialsMetaInput = { provider: string; }; -/* Mirror of autogpt_libs/supabase_integration_credentials_store/types.py:_BaseCredentials */ +/* Mirror of backend/backend/data/model.py:_BaseCredentials */ type BaseCredentials = { id: string; type: CredentialsType; @@ -297,7 +297,7 @@ type BaseCredentials = { provider: CredentialsProviderName; }; -/* Mirror of autogpt_libs/supabase_integration_credentials_store/types.py:OAuth2Credentials */ +/* Mirror of backend/backend/data/model.py:OAuth2Credentials */ export type OAuth2Credentials = BaseCredentials & { type: "oauth2"; scopes: string[]; @@ -309,7 +309,7 @@ export type OAuth2Credentials = BaseCredentials & { metadata: Record; }; -/* Mirror of autogpt_libs/supabase_integration_credentials_store/types.py:APIKeyCredentials */ +/* Mirror of backend/backend/data/model.py:APIKeyCredentials */ export type APIKeyCredentials = BaseCredentials & { type: "api_key"; title: string; diff --git a/docs/content/platform/getting-started.md b/docs/content/platform/getting-started.md index b57ff7de8f63..3613d014f5cf 100644 --- a/docs/content/platform/getting-started.md +++ b/docs/content/platform/getting-started.md @@ -128,3 +128,21 @@ By default the application for different services run on the following ports: Frontend UI Server: 3000 Backend Websocket Server: 8001 Execution API Rest Server: 8006 + +#### Additional Notes + +You may want to change your encryption key in the `.env` file in the `autogpt_platform/backend` directory. + +To generate a new encryption key, run the following command in python: + +```python +from cryptography.fernet import Fernet;Fernet.generate_key().decode() +``` + +Or run the following command in the `autogpt_platform/backend` directory: + +```bash +poetry run cli gen-encrypt-key +``` + +Then, replace the existing key in the `autogpt_platform/backend/.env` file with the new one. diff --git a/docs/content/platform/new_blocks.md b/docs/content/platform/new_blocks.md index faad5e8a9d78..623b1d9b5257 100644 --- a/docs/content/platform/new_blocks.md +++ b/docs/content/platform/new_blocks.md @@ -113,7 +113,7 @@ Implementing the block itself is relatively simple. On top of the instructions a you're going to add a `credentials` parameter to the `Input` model and the `run` method: ```python -from autogpt_libs.supabase_integration_credentials_store.types import ( +from backend.data.model import ( APIKeyCredentials, OAuth2Credentials, Credentials,