From 05801808efa9c27d1e13c0f64469ac9693104659 Mon Sep 17 00:00:00 2001 From: Joshua Knight Date: Mon, 8 Jul 2024 07:57:04 +0100 Subject: [PATCH 01/13] Adding initial test suite --- .github/workflows/run-test-suite.yml | 76 ++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .github/workflows/run-test-suite.yml diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml new file mode 100644 index 0000000..0ad4f24 --- /dev/null +++ b/.github/workflows/run-test-suite.yml @@ -0,0 +1,76 @@ +--- +name: aep +on: + pull_request: + branches: + - 'main' + + + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.10', '3.11', '3.12'] + + env: + AZURE_VAULT_ID: ${{ secrets.TEST_AZURE_VAULT_ID }} + AZURE_CLIENT_ID: ${{ secrets.TEST_AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.TEST_AZURE_TENANT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.TEST_AZURE_CLIENT_SECRET }} + AZURE_OPENAI_ENDPOINT: ${{ secrets.TEST_AZURE_OPENAI_ENDPOINT }} + AZURE_OPENAI_KEY: ${{ secrets.TEST_AZURE_OPENAI_KEY }} + OPENAI_API_KEY: ${{ secrets.TEST_AZURE_OPENAI_KEY }} + AZURE_CS_ENDPOINT: ${{ secrets.TEST_AZURE_CS_ENDPOINT }} + AZURE_CS_KEY: ${{ secrets.TEST_AZURE_CS_KEY }} + + SYSTEM_PROMPT_FILE: "system_prompts/prompts.json" + azure_openai_api_version: "2023-12-01-preview" + SYSTEM_API_KEY: "system" + OPENAI_API_TYPE: "azure" + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f src/requirements.txt ]; then pip install -r src/requirements.txt; fi + + - name: Run tests + working-directory: src/ + run: | + pytest tests/ -vv -s --junit-xml=test-results.xml + + - name: Surface failing tests + if: always() + uses: pmeier/pytest-results-action@main + with: + # A list of JUnit XML files, directories containing the former, and wildcard + # patterns to process. + # See @actions/glob for supported patterns. + path: src/test-results.xml + + # (Optional) Add a summary of the results at the top of the report + summary: true + + # (Optional) Select which results should be included in the report. + # Follows the same syntax as `pytest -r` + display-options: fEX + + # (Optional) Fail the workflow if no JUnit XML was found. + fail-on-empty: true + + # (Optional) Title of the test results section in the workflow summary + title: AEP Test Results + + env: + CI: true From 226513494194f73348852b5f58d6615b397e6883 Mon Sep 17 00:00:00 2001 From: Joshua Knight Date: Mon, 8 Jul 2024 08:31:02 +0100 Subject: [PATCH 02/13] Adding more logging around prompt detection --- src/helpers/prompts.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/helpers/prompts.py b/src/helpers/prompts.py index a554489..12fe03a 100644 --- a/src/helpers/prompts.py +++ b/src/helpers/prompts.py @@ -89,8 +89,9 @@ def reduce_prompt_tokens(prompt): def check_for_prompt_inj(prompt): - + event_logger.debug(f"Checking for prompt injection") url = config.azure_cs_endpoint + "/contentsafety/text:shieldPrompt?api-version=2024-02-15-preview" + event_logger.debug(f"CS Config URL: {url}") headers = { 'Ocp-Apim-Subscription-Key': config.azure_cs_key, 'Content-Type': 'application/json' @@ -101,15 +102,20 @@ def check_for_prompt_inj(prompt): f"{prompt}" ] } - response = requests.post(url, headers=headers, data=json.dumps(data)) + try: + response = requests.post(url, headers=headers, data=json.dumps(data)) + event_logger.debug(f"Response from AI ContentSafety: {response.json()}") - # Log the response - response_json = response.json() + # Log the response + response_json = response.json() - # Check if attackDetected is True in either userPromptAnalysis or documentsAnalysis - if response_json['documentsAnalysis'][0]['attackDetected']: - event_logger.info(f"Response from AI ContentSafety: {response.json()}") - event_logger.info(f"Prompt injection Detected in: {prompt}") - return False # Fail if attackDetected is True + # Check if attackDetected is True in either userPromptAnalysis or documentsAnalysis + if response_json['documentsAnalysis'][0]['attackDetected']: + event_logger.info(f"Response from AI ContentSafety: {response.json()}") + event_logger.info(f"Prompt injection Detected in: {prompt}") + return False # Fail if attackDetected is True + + except Exception as err: + event_logger.error(f"Failed to perform prompt injection detection: {err}") return True From e89f234ff5bca2518abe648c5fe2170e6fe5c14d Mon Sep 17 00:00:00 2001 From: Joshua Knight Date: Mon, 8 Jul 2024 09:13:57 +0100 Subject: [PATCH 03/13] Adding make unittest --- .github/workflows/run-test-suite.yml | 12 +++--------- Makefile | 6 +++++- src/requirements.txt | 1 + 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index 0ad4f24..9287cb1 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.10', '3.11', '3.12'] + python-version: ['3.12'] env: AZURE_VAULT_ID: ${{ secrets.TEST_AZURE_VAULT_ID }} @@ -40,15 +40,9 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - if [ -f src/requirements.txt ]; then pip install -r src/requirements.txt; fi - - name: Run tests - working-directory: src/ - run: | - pytest tests/ -vv -s --junit-xml=test-results.xml + working-directory: / + run: make unittest - name: Surface failing tests if: always() diff --git a/Makefile b/Makefile index 5bc7dfd..0f085c7 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,10 @@ mkdocs-build: run: ./buildscripts/run_docker.sh +unittest: + $(MAKE) build-local + $(MAKE) run-tests + run-tests: ./buildscripts/run_tests.sh @@ -56,4 +60,4 @@ inference-build: ./buildscripts/build_inference_service.sh inference-run: - ./buildscripts/run_inference_service.sh \ No newline at end of file + ./buildscripts/run_inference_service.sh diff --git a/src/requirements.txt b/src/requirements.txt index b242aca..4b5dc32 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -12,3 +12,4 @@ prometheus_fastapi_instrumentator pydantic-core pytest markdown +coverage From 14a8b8867186ee0ddecfa92f8711f2ab2258e11c Mon Sep 17 00:00:00 2001 From: Joshua Knight Date: Mon, 8 Jul 2024 09:16:30 +0100 Subject: [PATCH 04/13] Removing WorkingDir --- .github/workflows/run-test-suite.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index 9287cb1..8a6a215 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -41,7 +41,6 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run tests - working-directory: / run: make unittest - name: Surface failing tests From 14b464aa38bece76c0768856aa29c015f155f6f7 Mon Sep 17 00:00:00 2001 From: Joshua Knight Date: Mon, 8 Jul 2024 09:29:36 +0100 Subject: [PATCH 05/13] including .env --- .github/workflows/run-test-suite.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index 8a6a215..68da756 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -41,7 +41,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run tests - run: make unittest + run: | + include .env + make unittest - name: Surface failing tests if: always() From 330be66a325d4d5baa8f71fcec8419acffdd5c37 Mon Sep 17 00:00:00 2001 From: Joshua Knight Date: Mon, 8 Jul 2024 09:33:59 +0100 Subject: [PATCH 06/13] formatting fix --- .github/workflows/run-test-suite.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index 68da756..b738d8e 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -42,8 +42,8 @@ jobs: - name: Run tests run: | - include .env - make unittest + include .env + make unittest - name: Surface failing tests if: always() From c05a8c0a9e1049f01306ee00590d8f567a2ae8c2 Mon Sep 17 00:00:00 2001 From: Joshua Knight Date: Mon, 8 Jul 2024 09:41:50 +0100 Subject: [PATCH 07/13] Echo envs to .env --- .github/workflows/run-test-suite.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index b738d8e..d96a27a 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -25,11 +25,24 @@ jobs: OPENAI_API_KEY: ${{ secrets.TEST_AZURE_OPENAI_KEY }} AZURE_CS_ENDPOINT: ${{ secrets.TEST_AZURE_CS_ENDPOINT }} AZURE_CS_KEY: ${{ secrets.TEST_AZURE_CS_KEY }} - SYSTEM_PROMPT_FILE: "system_prompts/prompts.json" azure_openai_api_version: "2023-12-01-preview" SYSTEM_API_KEY: "system" OPENAI_API_TYPE: "azure" + run: | + echo "AZURE_VAULT_ID=${AZURE_VAULT_ID}" >> .env + echo "AZURE_CLIENT_ID=${AZURE_CLIENT_ID}" >> .env + echo "AZURE_TENANT_ID=${AZURE_TENANT_ID}" >> .env + echo "AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET}" >> .env + echo "AZURE_OPENAI_ENDPOINT=${AZURE_OPENAI_ENDPOINT}" >> .env + echo "AZURE_OPENAI_KEY=${AZURE_OPENAI_KEY}" >> .env + echo "OPENAI_API_KEY=${OPENAI_API_KEY}" >> .env + echo "AZURE_CS_ENDPOINT=${AZURE_CS_ENDPOINT}" >> .env + echo "AZURE_CS_KEY=${AZURE_CS_KEY}" >> .env + echo "SYSTEM_PROMPT_FILE=${SYSTEM_PROMPT_FILE}" >> .env + echo "azure_openai_api_version=${azure_openai_api_version}" >> .env + echo "SYSTEM_API_KEY=${SYSTEM_API_KEY}" >> .env + echo "OPENAI_API_TYPE=${OPENAI_API_TYPE}" >> .env steps: - name: Checkout repository @@ -42,7 +55,6 @@ jobs: - name: Run tests run: | - include .env make unittest - name: Surface failing tests From 8df4a3466c0cffcfafa6895538f93cc12a3aade9 Mon Sep 17 00:00:00 2001 From: Joshua Knight Date: Mon, 8 Jul 2024 09:44:18 +0100 Subject: [PATCH 08/13] Moving create .env step --- .github/workflows/run-test-suite.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index d96a27a..dba5af0 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -29,7 +29,11 @@ jobs: azure_openai_api_version: "2023-12-01-preview" SYSTEM_API_KEY: "system" OPENAI_API_TYPE: "azure" - run: | + + steps: + + - name: Create .env file + run: | echo "AZURE_VAULT_ID=${AZURE_VAULT_ID}" >> .env echo "AZURE_CLIENT_ID=${AZURE_CLIENT_ID}" >> .env echo "AZURE_TENANT_ID=${AZURE_TENANT_ID}" >> .env @@ -43,8 +47,7 @@ jobs: echo "azure_openai_api_version=${azure_openai_api_version}" >> .env echo "SYSTEM_API_KEY=${SYSTEM_API_KEY}" >> .env echo "OPENAI_API_TYPE=${OPENAI_API_TYPE}" >> .env - - steps: + - name: Checkout repository uses: actions/checkout@v2 From 5d637025e32a6bb90efc888ffc7460c4f6681763 Mon Sep 17 00:00:00 2001 From: Joshua Knight Date: Mon, 8 Jul 2024 09:46:44 +0100 Subject: [PATCH 09/13] Echo pwd --- .github/workflows/run-test-suite.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index dba5af0..f1ce35b 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -34,6 +34,7 @@ jobs: - name: Create .env file run: | + pwd echo "AZURE_VAULT_ID=${AZURE_VAULT_ID}" >> .env echo "AZURE_CLIENT_ID=${AZURE_CLIENT_ID}" >> .env echo "AZURE_TENANT_ID=${AZURE_TENANT_ID}" >> .env From 873bb86a6bf79ab9d604e9686d67be2bc0e1fe1e Mon Sep 17 00:00:00 2001 From: Joshua Knight Date: Mon, 8 Jul 2024 09:47:58 +0100 Subject: [PATCH 10/13] Echo pwd --- .github/workflows/run-test-suite.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index f1ce35b..7124c74 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -59,6 +59,7 @@ jobs: - name: Run tests run: | + pwd make unittest - name: Surface failing tests From bbb19df7178ca8869aa9b57708deccc76dcb19a8 Mon Sep 17 00:00:00 2001 From: Joshua Knight Date: Mon, 8 Jul 2024 09:57:37 +0100 Subject: [PATCH 11/13] Echo pwd --- .github/workflows/run-test-suite.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index 7124c74..b2694af 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -35,19 +35,19 @@ jobs: - name: Create .env file run: | pwd - echo "AZURE_VAULT_ID=${AZURE_VAULT_ID}" >> .env - echo "AZURE_CLIENT_ID=${AZURE_CLIENT_ID}" >> .env - echo "AZURE_TENANT_ID=${AZURE_TENANT_ID}" >> .env - echo "AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET}" >> .env - echo "AZURE_OPENAI_ENDPOINT=${AZURE_OPENAI_ENDPOINT}" >> .env - echo "AZURE_OPENAI_KEY=${AZURE_OPENAI_KEY}" >> .env - echo "OPENAI_API_KEY=${OPENAI_API_KEY}" >> .env - echo "AZURE_CS_ENDPOINT=${AZURE_CS_ENDPOINT}" >> .env - echo "AZURE_CS_KEY=${AZURE_CS_KEY}" >> .env - echo "SYSTEM_PROMPT_FILE=${SYSTEM_PROMPT_FILE}" >> .env - echo "azure_openai_api_version=${azure_openai_api_version}" >> .env - echo "SYSTEM_API_KEY=${SYSTEM_API_KEY}" >> .env - echo "OPENAI_API_TYPE=${OPENAI_API_TYPE}" >> .env + echo "AZURE_VAULT_ID=${AZURE_VAULT_ID}" >> ./.env + echo "AZURE_CLIENT_ID=${AZURE_CLIENT_ID}" >> ./.env + echo "AZURE_TENANT_ID=${AZURE_TENANT_ID}" >> ./.env + echo "AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET}" >> ./.env + echo "AZURE_OPENAI_ENDPOINT=${AZURE_OPENAI_ENDPOINT}" >> ./.env + echo "AZURE_OPENAI_KEY=${AZURE_OPENAI_KEY}" >> ./.env + echo "OPENAI_API_KEY=${OPENAI_API_KEY}" >> ./.env + echo "AZURE_CS_ENDPOINT=${AZURE_CS_ENDPOINT}" >> ./.env + echo "AZURE_CS_KEY=${AZURE_CS_KEY}" >> ./.env + echo "SYSTEM_PROMPT_FILE=${SYSTEM_PROMPT_FILE}" >> ./.env + echo "azure_openai_api_version=${azure_openai_api_version}" >> ./.env + echo "SYSTEM_API_KEY=${SYSTEM_API_KEY}" >> ./.env + echo "OPENAI_API_TYPE=${OPENAI_API_TYPE}" >> ./.env - name: Checkout repository uses: actions/checkout@v2 From 1e2bbff225f0c4d0bfdf701fe7b144e340971708 Mon Sep 17 00:00:00 2001 From: Joshua Knight Date: Mon, 8 Jul 2024 10:01:10 +0100 Subject: [PATCH 12/13] Moving .env creation to make step --- .github/workflows/run-test-suite.yml | 30 ++++++++++++---------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index b2694af..fa46f6f 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -32,23 +32,6 @@ jobs: steps: - - name: Create .env file - run: | - pwd - echo "AZURE_VAULT_ID=${AZURE_VAULT_ID}" >> ./.env - echo "AZURE_CLIENT_ID=${AZURE_CLIENT_ID}" >> ./.env - echo "AZURE_TENANT_ID=${AZURE_TENANT_ID}" >> ./.env - echo "AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET}" >> ./.env - echo "AZURE_OPENAI_ENDPOINT=${AZURE_OPENAI_ENDPOINT}" >> ./.env - echo "AZURE_OPENAI_KEY=${AZURE_OPENAI_KEY}" >> ./.env - echo "OPENAI_API_KEY=${OPENAI_API_KEY}" >> ./.env - echo "AZURE_CS_ENDPOINT=${AZURE_CS_ENDPOINT}" >> ./.env - echo "AZURE_CS_KEY=${AZURE_CS_KEY}" >> ./.env - echo "SYSTEM_PROMPT_FILE=${SYSTEM_PROMPT_FILE}" >> ./.env - echo "azure_openai_api_version=${azure_openai_api_version}" >> ./.env - echo "SYSTEM_API_KEY=${SYSTEM_API_KEY}" >> ./.env - echo "OPENAI_API_TYPE=${OPENAI_API_TYPE}" >> ./.env - - name: Checkout repository uses: actions/checkout@v2 @@ -60,6 +43,19 @@ jobs: - name: Run tests run: | pwd + echo "AZURE_VAULT_ID=${AZURE_VAULT_ID}" >> .env + echo "AZURE_CLIENT_ID=${AZURE_CLIENT_ID}" >> .env + echo "AZURE_TENANT_ID=${AZURE_TENANT_ID}" >> .env + echo "AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET}" >> .env + echo "AZURE_OPENAI_ENDPOINT=${AZURE_OPENAI_ENDPOINT}" >> .env + echo "AZURE_OPENAI_KEY=${AZURE_OPENAI_KEY}" >> .env + echo "OPENAI_API_KEY=${OPENAI_API_KEY}" >> .env + echo "AZURE_CS_ENDPOINT=${AZURE_CS_ENDPOINT}" >> .env + echo "AZURE_CS_KEY=${AZURE_CS_KEY}" >> .env + echo "SYSTEM_PROMPT_FILE=${SYSTEM_PROMPT_FILE}" >> .env + echo "azure_openai_api_version=${azure_openai_api_version}" >> .env + echo "SYSTEM_API_KEY=${SYSTEM_API_KEY}" >> .env + echo "OPENAI_API_TYPE=${OPENAI_API_TYPE}" >> .env make unittest - name: Surface failing tests From 9c592fe1feb5dbd0f2c94a506cdd0ad5c53eedfd Mon Sep 17 00:00:00 2001 From: Joshua Knight Date: Mon, 8 Jul 2024 10:03:45 +0100 Subject: [PATCH 13/13] Fail on Empty -> false --- .github/workflows/run-test-suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index fa46f6f..f083755 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -75,7 +75,7 @@ jobs: display-options: fEX # (Optional) Fail the workflow if no JUnit XML was found. - fail-on-empty: true + fail-on-empty: false # (Optional) Title of the test results section in the workflow summary title: AEP Test Results