Merge branch 'main' into feature/sidebar-label

.github/workflows/ci.yaml

@@ -11,15 +11,18 @@ jobs:
         python-version: ['3.11']
 
     steps:
-      - uses: actions/checkout@v2
+      - name: Check out repository
+        uses: actions/checkout@v4
+
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install requirements
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements-dev.txt
+          pip install pre-commit
+
       - name: Run Pre commit hook (formatting, linting & tests)
         run: pre-commit run --all-files --hook-stage pre-push --show-diff-on-failure

.github/workflows/deploy_docs.yaml

@@ -19,8 +19,8 @@ jobs:
       - name: Install requirements
         run: |
           python -m pip install --upgrade pip
-          pip install mkdocs mkdocs-techdocs-core pymdown-extensions mkdocs_monorepo_plugin
+          pip install mkdocs pymdown-extensions termynal mkdocs-material
       - name: Deploying MkDocs documentation
         run: |
           mkdocs build
-          mkdocs gh-deploy --force
+          mkdocs gh-deploy --force

.pre-commit-config.yaml

@@ -7,22 +7,17 @@ repos:
       - id: check-toml
       - id: check-json
       - id: check-added-large-files
-  - repo: local
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.1
     hooks:
-      - id: isort
-        name: Sorting imports (isort)
-        entry: isort
-        types: [python]
-        language: system
       - id: ruff
-        name: Linting (ruff)
-        entry: ruff --fix
-        types: [python]
-        language: system
+        args: [ --fix ]
+      - id: ruff-format
+
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.7.1
+    hooks:
       - id: nbstripout
-        name: Strip Jupyter notebook output (nbstripout)
-        entry: nbstripout
-        types: [file]
-        files: (.ipynb)$
-        language: system
+
 exclude: ^(.svn|CVS|.bzr|.hg|.git|__pycache__|.tox|.ipynb_checkpoints|assets|tests/assets/|venv/|.venv/)

.skaff/skaff.yaml

@@ -7,7 +7,7 @@ name: GenAI RAG indus kit
 owner: [email protected]
 description: >
   Deploy production grade RAGs quickly, on any cloud.
-documentation_url: https://artefactory.github.io/skaff-rag-accelerator/
+documentation_url: https://artefactory-skaff.github.io/skaff-rag-accelerator/
 
 type: deployable   # deployable, knowldege pack
 lifecycle: prototype  # prototype, production

README.md

@@ -5,7 +5,7 @@ This is a starter kit to deploy a modularizable RAG locally or on the cloud (or
 
 ## Features
 
-- A configurable RAG setup based around Langchain ([Check out the configuration cookbook here](https://artefactory.github.io/skaff-rag-accelerator/cookbook/))
+- A configurable RAG setup based around Langchain ([Check out the configuration cookbook here](https://artefactory-skaff.github.io/skaff-rag-accelerator/cookbook/))
 - `RAG` and `RagConfig` python classes that manage components (vector store, llm, retreiver, ...)
 - A REST API based on Langserve + FastAPI to provide easy access to the RAG as a web backend
 - Optional API plugins for secure user authentication, session management, ...
@@ -17,7 +17,7 @@ This is a starter kit to deploy a modularizable RAG locally or on the cloud (or
 
 ## Quickstart
 
-This quickstart will guide you through the steps to serve the RAG and load a few documents. 
+This quickstart will guide you through the steps to serve the RAG and load a few documents.
 
 You will run both the back and front on your machine.
 
@@ -80,7 +80,7 @@ Right now the RAG does not have any documents loaded, you can use the notebook i
 
 To deep dive into under the hood, take a look at the documentation
 
-[On github pages](https://artefactory.github.io/skaff-rag-accelerator/)
+[On github pages](https://artefactory-skaff.github.io/skaff-rag-accelerator/)
 
 Or serve them locally:
 ```shell

backend/Dockerfile

@@ -21,4 +21,4 @@ EXPOSE $PORT
 
 COPY . ./backend
 
-CMD python -m uvicorn backend.main:app --host 0.0.0.0 --port $PORT
+CMD python -m uvicorn backend.main:app --host 0.0.0.0 --port $PORT

backend/api_plugins/__init__.py

@@ -1,3 +1,13 @@
-from backend.api_plugins.insecure_authentication.insecure_authentication import insecure_authentication_routes
-from backend.api_plugins.secure_authentication.secure_authentication import authentication_routes
+from backend.api_plugins.insecure_authentication.insecure_authentication import (
+    insecure_authentication_routes,
+)
+from backend.api_plugins.secure_authentication.secure_authentication import (
+    authentication_routes,
+)
 from backend.api_plugins.sessions.sessions import session_routes
+
+__all__ = [
+    "insecure_authentication_routes",
+    "authentication_routes",
+    "session_routes",
+]

backend/api_plugins/insecure_authentication/insecure_authentication.py

@@ -25,23 +25,30 @@ async def get_current_user(email: str) -> User:
     async def signup(email: str) -> dict:
         user = User(email=email)
         if user_exists(user.email):
-            raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"User {user.email} already registered")
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=f"User {user.email} already registered",
+            )
         create_user(user)
         return {"email": user.email}
 
-
     @app.delete("/user/")
     async def del_user(current_user: User = Depends(get_current_user)) -> dict:
         email = current_user.email
         try:
             user = get_user(email)
             if user is None:
-                raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"User {email} not found")
+                raise HTTPException(
+                    status_code=status.HTTP_404_NOT_FOUND,
+                    detail=f"User {email} not found",
+                )
             delete_user(email)
             return {"detail": f"User {email} deleted"}
         except Exception:
-            raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal Server Error")
-
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail="Internal Server Error",
+            )
 
     @app.post("/user/login")
     async def login(email: str) -> dict:
@@ -51,16 +58,20 @@ async def login(email: str) -> dict:
                 status_code=status.HTTP_401_UNAUTHORIZED,
                 detail="Incorrect username",
             )
-        return {"access_token": email, "token_type": "bearer"}  # Fake bearer token to still provide user authentication
-
+        return {
+            "access_token": email,
+            "token_type": "bearer",
+        }  # Fake bearer token to still provide user authentication
 
     @app.get("/user/me")
     async def user_me(current_user: User = Depends(get_current_user)) -> User:
         return current_user
 
-
     @app.get("/user")
     async def user_root() -> dict:
-        return Response("Insecure user management routes are enabled. Do not use in prod.", status_code=200)
+        return Response(
+            "Insecure user management routes are enabled. Do not use in prod.",
+            status_code=200,
+        )
 
     return Depends(get_current_user)

backend/api_plugins/lib/user_management.py

@@ -13,6 +13,7 @@ class UnsecureUser(BaseModel):
     email: str = None
     password: bytes = None
 
+
 class User(BaseModel):
     email: str = None
     hashed_password: str = None
@@ -26,7 +27,8 @@ def from_unsecure_user(cls, unsecure_user: UnsecureUser):
 def create_user(user: User) -> None:
     with Database() as connection:
         connection.execute(
-            "INSERT INTO users (email, password) VALUES (?, ?)", (user.email, user.hashed_password)
+            "INSERT INTO users (email, password) VALUES (?, ?)",
+            (user.email, user.hashed_password),
         )
 
 
@@ -54,12 +56,17 @@ def authenticate_user(username: str, password: bytes) -> bool | User:
     if not user:
         return False
 
-    if argon2.verify_password(user.hashed_password.encode("utf-8"), password.encode("utf-8")):
+    if argon2.verify_password(
+        user.hashed_password.encode("utf-8"), password.encode("utf-8")
+    ):
         return user
 
     return False
 
-def create_access_token(*, data: dict, expires_delta: Optional[timedelta] = None) -> str:
+
+def create_access_token(
+    *, data: dict, expires_delta: Optional[timedelta] = None
+) -> str:
     to_encode = data.copy()
     if expires_delta:
         expire = datetime.utcnow() + expires_delta

backend/api_plugins/secure_authentication/secure_authentication.py

@@ -22,6 +22,7 @@
 
 def authentication_routes(app, dependencies=List[Depends]):
     from backend.database import Database
+
     with Database() as connection:
         connection.run_script(Path(__file__).parent / "users_tables.sql")
 
@@ -46,39 +47,41 @@ async def get_current_user(token: str = Depends(oauth2_scheme)) -> User:
         except JWTError:
             raise credentials_exception
 
-
     @app.post("/user/signup", include_in_schema=ADMIN_MODE)
     async def signup(user: UnsecureUser) -> dict:
         if not ADMIN_MODE:
-            raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Signup is disabled")
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail="Signup is disabled"
+            )
 
         user = User.from_unsecure_user(user)
         if user_exists(user.email):
             raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST, detail=f"User {user.email} already registered"
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=f"User {user.email} already registered",
             )
 
         create_user(user)
         return {"email": user.email}
 
-
     @app.delete("/user/")
     async def del_user(current_user: User = Depends(get_current_user)) -> dict:
         email = current_user.email
         try:
             user = get_user(email)
             if user is None:
                 raise HTTPException(
-                    status_code=status.HTTP_404_NOT_FOUND, detail=f"User {email} not found"
+                    status_code=status.HTTP_404_NOT_FOUND,
+                    detail=f"User {email} not found",
                 )
             delete_user(email)
             return {"detail": f"User {email} deleted"}
         except Exception:
             raise HTTPException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal Server Error"
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail="Internal Server Error",
             )
 
-
     @app.post("/user/login")
     async def login(form_data: OAuth2PasswordRequestForm = Depends()) -> dict:
         user = authenticate_user(form_data.username, form_data.password)
@@ -93,15 +96,12 @@ async def login(form_data: OAuth2PasswordRequestForm = Depends()) -> dict:
         access_token = create_access_token(data=user_data)
         return {"access_token": access_token, "token_type": "bearer"}
 
-
     @app.get("/user/me")
     async def user_me(current_user: User = Depends(get_current_user)) -> User:
         return current_user
 
-
     @app.get("/user")
     async def user_root() -> dict:
         return Response("User management routes are enabled.", status_code=200)
 
-
     return Depends(get_current_user)

backend/config.py

@@ -12,55 +12,62 @@
 
 load_dotenv()
 
+
 @dataclass
 class LLMConfig:
     source: BaseChatModel | LLM | str
     source_config: dict
 
+
 @dataclass
 class VectorStoreConfig:
     source: VectorStore | str
     source_config: dict
 
     insertion_mode: str  # "None", "full", "incremental"
 
+
 @dataclass
 class EmbeddingModelConfig:
     source: Embeddings | str
     source_config: dict
 
+
 @dataclass
 class DatabaseConfig:
     database_url: str
 
+
 @dataclass
 class RagConfig:
     """
     Configuration class for the Retrieval-Augmented Generation (RAG) system.
     It is meant to be injected in the RAG class to configure the various components.
 
-    This class holds the configuration for the various components that make up the RAG system, including
-    the language model, vector store, embedding model, and database configurations. It provides a method
-    to construct a RagConfig instance from a YAML file, allowing for easy external configuration.
+    This class holds the configuration for the various components that make up the RAG
+    system, including the language model, vector store, embedding model, and database
+    configurations. It provides a method to construct a RagConfig instance from a YAML
+    file, allowing for easy external configuration.
 
     Attributes:
         llm (LLMConfig): Configuration for the language model component.
         vector_store (VectorStoreConfig): Configuration for the vector store component.
-        embedding_model (EmbeddingModelConfig): Configuration for the embedding model component.
+        embedding_model (EmbeddingModelConfig): Configuration for the embedding model
+            component.
         database (DatabaseConfig): Configuration for the database connection.
 
     Methods:
-        from_yaml: Class method to create an instance of RagConfig from a YAML file, with optional environment
-                   variables for template rendering.
+        from_yaml: Class method to create an instance of RagConfig from a YAML file,
+            with optional environment variables for template rendering.
     """
 
-    llm:                      LLMConfig             = field(default_factory=LLMConfig)
-    vector_store:             VectorStoreConfig     = field(default_factory=VectorStoreConfig)
-    embedding_model:          EmbeddingModelConfig  = field(default_factory=EmbeddingModelConfig)
-    database:                 DatabaseConfig        = field(default_factory=DatabaseConfig)
-    chat_history_window_size: int                   = 5
-    max_tokens_limit:         int                   = 3000
-    response_mode:            str                   = None
+    llm: LLMConfig = field(default_factory=LLMConfig)
+    vector_store: VectorStoreConfig = field(default_factory=VectorStoreConfig)
+    embedding_model: EmbeddingModelConfig = field(default_factory=EmbeddingModelConfig)
+    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    chat_history_window_size: int = 5
+    max_tokens_limit: int = 3000
+    response_mode: str = None
 
     @classmethod
     def from_yaml(cls, yaml_path: Path, env: dict = None):