merged release/kms

.env.example

@@ -54,6 +54,8 @@ EXTERNAL_SUPABASE_URL=http://localhost:54321
 SUPABASE_SERVICE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU
 PG_DATABASE_URL=postgresql://postgres:[email protected]:54322/postgres
 PG_DATABASE_ASYNC_URL=postgresql+asyncpg://postgres:[email protected]:54322/postgres
+SQLALCHEMY_POOL_SIZE=10
+SQLALCHEMY_MAX_POOL_OVERFLOW=0
 JWT_SECRET_KEY=super-secret-jwt-token-with-at-least-32-characters-long
 AUTHENTICATE=true
 TELEMETRY_ENABLED=true
@@ -64,6 +66,13 @@ BACKEND_URL=http://localhost:5050
 EMBEDDING_DIM=1536
 DEACTIVATE_STRIPE=true
 
+
+# PARSEABLE LOGGING
+USE_PARSEABLE=False
+PARSEABLE_STREAM_NAME=quivr-api
+PARSEABLE_URL=<change-me>
+PARSEABLE_AUTH=<change-me>
+
 #RESEND
 RESEND_API_KEY=<change-me>
 RESEND_EMAIL_ADDRESS=[email protected]

.github/workflows/porter_stack_cdp-front.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+    - main
+name: Deploy to cdp-front
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Set Github tag
+      id: vars
+      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+    - name: Setup porter
+      uses: porter-dev/[email protected]
+    - name: Deploy stack
+      timeout-minutes: 30
+      run: exec porter apply
+      env:
+        PORTER_CLUSTER: "3877"
+        PORTER_DEPLOYMENT_TARGET_ID: cd21d246-86df-49e0-ba0a-78e2802572e7
+        PORTER_HOST: https://dashboard.porter.run
+        PORTER_PR_NUMBER: ${{ github.event.number }}
+        PORTER_PROJECT: "10983"
+        PORTER_REPO_NAME: ${{ github.event.repository.name }}
+        PORTER_STACK_NAME: cdp-front
+        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+        PORTER_TOKEN: ${{ secrets.PORTER_STACK_10983_3877 }}

.github/workflows/porter_stack_cdp.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+    - main
+name: Deploy to cdp
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Set Github tag
+      id: vars
+      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+    - name: Setup porter
+      uses: porter-dev/[email protected]
+    - name: Deploy stack
+      timeout-minutes: 30
+      run: exec porter apply
+      env:
+        PORTER_CLUSTER: "3877"
+        PORTER_DEPLOYMENT_TARGET_ID: cd21d246-86df-49e0-ba0a-78e2802572e7
+        PORTER_HOST: https://dashboard.porter.run
+        PORTER_PR_NUMBER: ${{ github.event.number }}
+        PORTER_PROJECT: "10983"
+        PORTER_REPO_NAME: ${{ github.event.repository.name }}
+        PORTER_STACK_NAME: cdp
+        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+        PORTER_TOKEN: ${{ secrets.PORTER_STACK_10983_3877 }}

.gitignore

@@ -80,7 +80,7 @@ paulgraham.py
 .env_test
 supabase/seed-airwallex.sql
 airwallexpayouts.py
-application.log
+**/application.log*
 backend/celerybeat-schedule.db
 
 backend/application.log.*
@@ -103,3 +103,4 @@ backend/core/examples/chatbot/.chainlit/translations/en-US.json
 .tox
 Pipfile
 *.pkl
+backend/benchmarks/data.json

.release-please-manifest.json

@@ -1,4 +1,4 @@
 {
     "backend/core": "0.0.16",
-    ".": "0.0.315"
+    ".": "0.0.319"
 }

CHANGELOG.md

@@ -1,5 +1,54 @@
 # Changelog
 
+## 0.0.319 (2024-10-01)
+
+## What's Changed
+* feat: Add error handling for rate-limited search in fetch_notion_pages by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3298
+* feat: Remove syncNotion from ConnectionCards by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3300
+
+
+**Full Changelog**: https://github.com/QuivrHQ/quivr/compare/v0.0.318...v0.0.319
+
+## 0.0.318 (2024-09-30)
+
+## What's Changed
+* feat: add syncNotion from ConnectionCards by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3292
+
+
+**Full Changelog**: https://github.com/QuivrHQ/quivr/compare/v0.0.317...v0.0.318
+
+## 0.0.317 (2024-09-30)
+
+## What's Changed
+* feat: update SyncsUser status field to be optional by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3284
+* fix(frontend): onboarding modal by @Zewed in https://github.com/QuivrHQ/quivr/pull/3286
+* feat: Remove syncNotion from ConnectionCards by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3289
+
+
+**Full Changelog**: https://github.com/QuivrHQ/quivr/compare/v0.0.316...v0.0.317
+
+## 0.0.316 (2024-09-30)
+
+## What's Changed
+* fix(core): enforce langchain <0.3 for pydantic v1 by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3217
+* chore(main): release core 0.0.16 by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3218
+* feat(assistants): mock api by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3195
+* chore(examples): fix chainlit example  by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3223
+* feat(integration): Notion by @chloedia in https://github.com/QuivrHQ/quivr/pull/3173
+* fix(migration): order migrations by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3231
+* feat: remove n+1 query knowledge by @AmineDiro in https://github.com/QuivrHQ/quivr/pull/3238
+* feat: introducing configurable retrieval workflows by @jacopo-chevallard in https://github.com/QuivrHQ/quivr/pull/3227
+* fix: wrong default path for CHAT_LLM_CONFIG_PATH env variable by @jacopo-chevallard in https://github.com/QuivrHQ/quivr/pull/3247
+* fix(frontend): sync folder for premium users by @Zewed in https://github.com/QuivrHQ/quivr/pull/3251
+* fix: correctly passing the prompt provided by the user by @jacopo-chevallard in https://github.com/QuivrHQ/quivr/pull/3252
+* fix: add sync user id aget_files by @AmineDiro in https://github.com/QuivrHQ/quivr/pull/3261
+* feat: update sync_user.py to include sync user id in aget_files call by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3276
+* chore: update Dockerfile dependencies and copy files by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3277
+* chore: add wget to Dockerfile dependencies by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3279
+
+
+**Full Changelog**: https://github.com/QuivrHQ/quivr/compare/v0.0.315...v0.0.316
+
 ## 0.0.315 (2024-09-17)
 
 ## What's Changed

backend/Dockerfile

@@ -3,22 +3,30 @@ FROM python:3.11.6-slim-bullseye
 WORKDIR /app
 
 # Install runtime dependencies
-RUN apt-get update && apt-get install -y \
+RUN apt-get clean && apt-get update && apt-get install -y \
     libgeos-dev \
     libcurl4-openssl-dev \
     libssl-dev \
     binutils \
-    pandoc \
     curl \
     git \
+    autoconf \
+    automake \
+    build-essential \
+    libtool \
+    python-dev \
+    build-essential \
+    wget \
+    # Additional dependencies for document handling
+    libmagic-dev \
     poppler-utils \
     tesseract-ocr \
-    libmagic-dev \
     libreoffice \
     libpq-dev \
     gcc \
-    wget \
-    && rm -rf /var/lib/apt/lists/*
+    libhdf5-serial-dev \
+    pandoc && \
+    rm -rf /var/lib/apt/lists/* && apt-get clean
 
 # Install Supabase CLI
 RUN ARCHITECTURE=$(uname -m) && \
@@ -32,7 +40,17 @@ RUN ARCHITECTURE=$(uname -m) && \
     rm supabase_1.163.6_linux_arm64.deb; \
     fi
 
-COPY . .
+COPY requirements.lock  pyproject.toml README.md ./
+COPY api/pyproject.toml api/README.md ./api/
+COPY api/quivr_api/__init__.py ./api/quivr_api/__init__.py
+COPY core/pyproject.toml core/README.md ./core/
+COPY core/quivr_core/__init__.py ./core/quivr_core/__init__.py
+COPY worker/pyproject.toml worker/README.md ./worker/
+COPY worker/quivr_worker/__init__.py ./worker/quivr_worker/__init__.py
+COPY worker/diff-assistant/pyproject.toml worker/diff-assistant/README.md ./worker/diff-assistant/
+COPY worker/diff-assistant/quivr_diff_assistant/__init__.py ./worker/diff-assistant/quivr_diff_assistant/__init__.py
+COPY core/MegaParse/pyproject.toml core/MegaParse/README.md ./core/MegaParse/
+COPY core/MegaParse/megaparse/__init__.py ./core/MegaParse/megaparse/__init__.py
 
 RUN PYTHONDONTWRITEBYTECODE=1 pip install --no-cache-dir -r requirements.lock
 
@@ -43,4 +61,5 @@ RUN playwright install --with-deps && \
 
 ENV PYTHONPATH=/app
 
+COPY . .
 EXPOSE 5050

backend/Dockerfile.dev

@@ -23,6 +23,7 @@ RUN apt-get clean && apt-get update && apt-get install -y \
     libreoffice \
     libpq-dev \
     gcc \
+    libhdf5-serial-dev \
     pandoc && \
     rm -rf /var/lib/apt/lists/* && apt-get clean
 
@@ -33,6 +34,8 @@ COPY core/pyproject.toml core/README.md ./core/
 COPY core/quivr_core/__init__.py ./core/quivr_core/__init__.py
 COPY worker/pyproject.toml worker/README.md ./worker/
 COPY worker/quivr_worker/__init__.py ./worker/quivr_worker/__init__.py
+COPY worker/diff-assistant/pyproject.toml worker/diff-assistant/README.md ./worker/diff-assistant/
+COPY worker/diff-assistant/quivr_diff_assistant/__init__.py ./worker/diff-assistant/quivr_diff_assistant/__init__.py
 COPY core/MegaParse/pyproject.toml core/MegaParse/README.md ./core/MegaParse/
 COPY core/MegaParse/megaparse/__init__.py ./core/MegaParse/megaparse/__init__.py