Refactor scripts to avoid anti-patterns, redundancy

app/backend/app.py

@@ -714,9 +714,10 @@ def create_app():
     # Log levels should be one of https://docs.python.org/3/library/logging.html#logging-levels
     # Set root level to WARNING to avoid seeing overly verbose logs from SDKS
     logging.basicConfig(level=logging.WARNING)
-    # Set the app logger level to INFO by default
-    default_level = "INFO"
-    app.logger.setLevel(os.getenv("APP_LOG_LEVEL", default_level))
+    # Set our own logger levels to INFO by default
+    app_level = os.getenv("APP_LOG_LEVEL", "INFO")
+    app.logger.setLevel(os.getenv("APP_LOG_LEVEL", app_level))
+    logging.getLogger("scripts").setLevel(app_level)
 
     if allowed_origin := os.getenv("ALLOWED_ORIGIN"):
         app.logger.info("ALLOWED_ORIGIN is set, enabling CORS for %s", allowed_origin)

app/backend/load_azd_env.py

@@ -0,0 +1,23 @@
+import json
+import logging
+import subprocess
+
+from dotenv import load_dotenv
+
+logger = logging.getLogger("scripts")
+
+
+def load_azd_env():
+    """Get path to current azd env file and load file using python-dotenv"""
+    result = subprocess.run("azd env list -o json", shell=True, capture_output=True, text=True)
+    if result.returncode != 0:
+        raise Exception("Error loading azd env")
+    env_json = json.loads(result.stdout)
+    env_file_path = None
+    for entry in env_json:
+        if entry["IsDefault"]:
+            env_file_path = entry["DotEnvPath"]
+    if not env_file_path:
+        raise Exception("No default azd env file found")
+    logger.info(f"Loading azd env from {env_file_path}")
+    load_dotenv(env_file_path, override=True)

app/backend/main.py

@@ -1,3 +1,12 @@
+import os
+
 from app import create_app
+from load_azd_env import load_azd_env
+
+# WEBSITE_HOSTNAME is always set by App Service, RUNNING_IN_PRODUCTION is set in main.bicep
+RUNNING_ON_AZURE = os.getenv("WEBSITE_HOSTNAME") is not None or os.getenv("RUNNING_IN_PRODUCTION") is not None
+
+if not RUNNING_ON_AZURE:
+    load_azd_env()
 
 app = create_app()

app/backend/prepdocslib/blobmanager.py

@@ -18,7 +18,7 @@
 
 from .listfilestrategy import File
 
-logger = logging.getLogger("ingester")
+logger = logging.getLogger("scripts")
 
 
 class BlobManager:

app/backend/prepdocslib/embeddings.py

@@ -17,7 +17,7 @@
 )
 from typing_extensions import TypedDict
 
-logger = logging.getLogger("ingester")
+logger = logging.getLogger("scripts")
 
 
 class EmbeddingBatch:

app/backend/prepdocslib/filestrategy.py

@@ -8,7 +8,7 @@
 from .searchmanager import SearchManager, Section
 from .strategy import DocumentAction, SearchInfo, Strategy
 
-logger = logging.getLogger("ingester")
+logger = logging.getLogger("scripts")
 
 
 async def parse_file(

app/backend/prepdocslib/htmlparser.py

@@ -7,7 +7,7 @@
 from .page import Page
 from .parser import Parser
 
-logger = logging.getLogger("ingester")
+logger = logging.getLogger("scripts")
 
 
 def cleanup_data(data: str) -> str:

app/backend/prepdocslib/integratedvectorizerstrategy.py

@@ -28,7 +28,7 @@
 from .searchmanager import SearchManager
 from .strategy import DocumentAction, SearchInfo, Strategy
 
-logger = logging.getLogger("ingester")
+logger = logging.getLogger("scripts")
 
 
 class IntegratedVectorizerStrategy(Strategy):

app/backend/prepdocslib/listfilestrategy.py

@@ -13,7 +13,7 @@
     DataLakeServiceClient,
 )
 
-logger = logging.getLogger("ingester")
+logger = logging.getLogger("scripts")
 
 
 class File:

app/backend/prepdocslib/pdfparser.py

@@ -11,7 +11,7 @@
 from .page import Page
 from .parser import Parser
 
-logger = logging.getLogger("ingester")
+logger = logging.getLogger("scripts")
 
 
 class LocalPdfParser(Parser):

app/backend/prepdocslib/searchmanager.py

@@ -26,7 +26,7 @@
 from .strategy import SearchInfo
 from .textsplitter import SplitPage
 
-logger = logging.getLogger("ingester")
+logger = logging.getLogger("scripts")
 
 
 class Section:

app/backend/prepdocslib/textsplitter.py

@@ -6,7 +6,7 @@
 
 from .page import Page, SplitPage
 
-logger = logging.getLogger("ingester")
+logger = logging.getLogger("scripts")
 
 
 class TextSplitter(ABC):

app/backend/requirements.in

@@ -29,3 +29,4 @@ beautifulsoup4
 types-beautifulsoup4
 msgraph-sdk==1.1.0
 openai-messages-token-helper
+python-dotenv

app/backend/requirements.txt

@@ -348,6 +348,8 @@ python-dateutil==2.9.0.post0
     #   microsoft-kiota-serialization-text
     #   pendulum
     #   time-machine
+python-dotenv==1.0.1
+    # via -r requirements.in
 quart==0.19.6
     # via
     #   -r requirements.in

app/frontend/src/pages/chat/Chat.tsx

@@ -131,7 +131,7 @@ const Chat = () => {
                 if (event["context"] && event["context"]["data_points"]) {
                     event["message"] = event["delta"];
                     askResponse = event as ChatAppResponse;
-                } else if (event["delta"]["content"]) {
+                } else if (event["delta"] && event["delta"]["content"]) {
                     setIsLoading(false);
                     await updateState(event["delta"]["content"]);
                 } else if (event["context"]) {

app/start.sh

@@ -1,21 +1,5 @@
 #!/bin/sh
 
-echo ""
-echo "Loading azd .env file from current environment"
-echo ""
-
-while IFS='=' read -r key value; do
-    value=$(echo "$value" | sed 's/^"//' | sed 's/"$//')
-    export "$key=$value"
-done <<EOF
-$(azd env get-values)
-EOF
-
-if [ $? -ne 0 ]; then
-    echo "Failed to load environment variables from azd environment"
-    exit $?
-fi
-
 cd ../
 echo 'Creating python virtual environment ".venv"'
 python3 -m venv .venv

docs/deploy_features.md

@@ -224,13 +224,13 @@ and will have ACLs associated with that directory. When the ingester runs, it wi
 If you are enabling this feature on an existing index, you should also update your index to have the new `storageUrl` field:
 
 ```shell
-./scripts/manageacl.ps1  -v --acl-action enable_acls
+python ./scripts/manageacl.py  -v --acl-action enable_acls
 ```
 
 And then update existing search documents with the storage URL of the main Blob container:
 
 ```shell
-./scripts/manageacl.ps1  -v --acl-action update_storage_urls --url <https://YOUR-MAIN-STORAGE-ACCOUNT.blob.core.windows.net/content/>
+python ./scripts/manageacl.py  -v --acl-action update_storage_urls --url <https://YOUR-MAIN-STORAGE-ACCOUNT.blob.core.windows.net/content/>
 ```
 
 Going forward, all uploaded documents will have their `storageUrl` set in the search index.