diff --git a/README-tests.md b/README-tests.md new file mode 100644 index 0000000000..3e8bd76653 --- /dev/null +++ b/README-tests.md @@ -0,0 +1,36 @@ +## Tests for code examples + +At Weaviate, we love automated tests. And we are making an effort to use automated tests to help us maintain the functionality of our code examples. + +**NOTE: This README and testing framework is a work in progress - if you see errors please let us know on GitHub, and please be patient with us.** + +### Implementation Overview + +Many of our in-doc [code examples](./README.md#code-examples) are pulled from external scripts. **These scripts are designed to be self-contained files that you can run.** + +In fact, tests are contained in-line in these scripts, in the form of simple assertions. + +The idea is that these scripts could then be executed from a central point, to make execution of the test as simple as possible. + +These tests are currently managed through [`pytest`](https://docs.pytest.org/), a testing framework for python. + +### Running tests + +To run the tests: + +- Install Python and Node. +- Install Docker (needed to spin up Weaviate) +- Set up API keys for Cohere, Hugging Face and OpenAI (needed for vectorization) as environment variables under `COHERE_APIKEY`, `HUGGINGFACE_APIKEY` and `OPENAI_APIKEY` respectively. +- Set up and activate the Python environment, and install the required libraries (`pytest`, `weaviate-client` and so on) +- We recommend doing this by running from the shell: +```bash +python -m venv .venv +source .venv/bin/python +pip install -r requirements.txt +``` +- Then run `pytest` from your environment. This will execute tests defined within the `tests` directory. + + +## Thanks + +A big thanks to Jeremy for suggesting this to us ;) diff --git a/README.md b/README.md index 6388421b17..7e1bf07c43 100644 --- a/README.md +++ b/README.md @@ -82,11 +82,13 @@ This makes use of our custom `FilteredTextBlock` JSX component. Here, the `FilteredTextBlock` component loads lines between the `startMarker` and `endMarker` from the imported scripts. This allows us to write complete scripts, which may include tests to reduce occurrences of erroneous code examples. +For more information about tests, please see [README-tests.md](./README-tests.md). + #### Legacy format In some code examples, the code will be written directly inside the `TabItem` component, as shown below. -```mdx +```md import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/_includes/code/howto/manage-data.multi-tenancy.py b/_includes/code/howto/manage-data.multi-tenancy.py index 4f2c8db9bf..d0be80f47d 100644 --- a/_includes/code/howto/manage-data.multi-tenancy.py +++ b/_includes/code/howto/manage-data.multi-tenancy.py @@ -12,7 +12,9 @@ ) class_name = 'MultiTenancyClass' # aka JeopardyQuestion -client.schema.delete_class(class_name) + +if client.schema.exists(class_name): + client.schema.delete_class(class_name) # ================================ @@ -48,9 +50,9 @@ ) # END ListTenants -# Test - tenants are returned in nondeterministic order -assert tenants == [Tenant(name='tenantA'), Tenant(name='tenantB')] or tenants == [Tenant(name='tenantB'), Tenant(name='tenantA')] - +# Test +assert Tenant(name='tenantA') in tenants +assert Tenant(name='tenantB') in tenants # ======================================= # ===== Remove tenants from a class ===== diff --git a/_includes/code/howto/read-all-objects.py b/_includes/code/howto/manage-data.read-all-objects.py similarity index 100% rename from _includes/code/howto/read-all-objects.py rename to _includes/code/howto/manage-data.read-all-objects.py diff --git a/_includes/code/howto/read-all-objects.ts b/_includes/code/howto/manage-data.read-all-objects.ts similarity index 100% rename from _includes/code/howto/read-all-objects.ts rename to _includes/code/howto/manage-data.read-all-objects.ts diff --git a/_includes/code/howto/manage-data.update.py b/_includes/code/howto/manage-data.update.py index ff13994ced..3b45528c74 100644 --- a/_includes/code/howto/manage-data.update.py +++ b/_includes/code/howto/manage-data.update.py @@ -33,8 +33,8 @@ # Clean slate if client.schema.exists('JeopardyQuestion'): client.schema.delete_class('JeopardyQuestion') -if not client.schema.exists('JeopardyQuestion'): - client.schema.create_class(class_definition) + +client.schema.create_class(class_definition) # ============================= @@ -117,7 +117,10 @@ # ============================= # DelProps START -def del_props(uuid: str, class_name: str, prop_names: [str]) -> None: +from typing import List +from weaviate import Client + +def del_props(client: Client, uuid: str, class_name: str, prop_names: List[str]) -> None: object_data = client.data_object.get(uuid, class_name=class_name) for prop_name in prop_names: if prop_name in object_data["properties"]: @@ -127,9 +130,11 @@ def del_props(uuid: str, class_name: str, prop_names: [str]) -> None: uuid = '...' # replace with the id of the object you want to delete properties from # DelProps END -uuid = result['id'] + +uuid = result['id'] # Actually get the ID for testing + # DelProps START -del_props(uuid, 'JeopardyQuestion', ['answer']) +del_props(client, uuid, 'JeopardyQuestion', ['answer']) # DelProps END # Test diff --git a/blog/2023-03-28-monitoring-weaviate-in-production/index.mdx b/blog/2023-03-28-monitoring-weaviate-in-production/index.mdx index 827bcb1641..58f0fa9396 100644 --- a/blog/2023-03-28-monitoring-weaviate-in-production/index.mdx +++ b/blog/2023-03-28-monitoring-weaviate-in-production/index.mdx @@ -46,7 +46,7 @@ If you are using Weaviate `1.17` or lower, you may want to upgrade to `1.18` bef ![monitoring weaviate in production](./img/Weaviate-monitoring-weaviate-in-prod-light.png#gh-light-mode-only) ![monitoring weaviate in production](./img/Weaviate-monitoring-weaviate-in-prod-dark.png#gh-dark-mode-only) -For the first approach we will use the open-source [Grafana agent](https://grafana.com/docs/grafana-cloud/data-configuration/agent/). In this case, we will show writing to Grafana Cloud for hosted metrics. This is configurable via the remote write section if you alternatively want to write to a self-hosted Mimir or Prometheus instance. +For the first approach we will use the open-source [Grafana agent](https://grafana.com/docs/agent/latest/). In this case, we will show writing to Grafana Cloud for hosted metrics. This is configurable via the remote write section if you alternatively want to write to a self-hosted Mimir or Prometheus instance. ### Steps to Install diff --git a/developers/weaviate/manage-data/read-all-objects.mdx b/developers/weaviate/manage-data/read-all-objects.mdx index 0001216b0f..88273cf718 100644 --- a/developers/weaviate/manage-data/read-all-objects.mdx +++ b/developers/weaviate/manage-data/read-all-objects.mdx @@ -12,8 +12,8 @@ import Badges from '/_includes/badges.mdx'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; -import PythonCode from '!!raw-loader!/_includes/code/howto/read-all-objects.py'; -import TSCode from '!!raw-loader!/_includes/code/howto/read-all-objects.ts'; +import PythonCode from '!!raw-loader!/_includes/code/howto/manage-data.read-all-objects.py'; +import TSCode from '!!raw-loader!/_includes/code/howto/manage-data.read-all-objects.ts'; ## Overview diff --git a/tests/docker-compose-anon.yml b/tests/docker-compose-anon.yml index b7864a001d..f1310803d2 100644 --- a/tests/docker-compose-anon.yml +++ b/tests/docker-compose-anon.yml @@ -9,7 +9,7 @@ services: - '8090' # Specify different port to avoid confusion - --scheme - http - image: semitechnologies/weaviate:1.19.6 + image: semitechnologies/weaviate:1.20.2 ports: - 8090:8090 restart: on-failure:0 diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 442fb0b11e..3ae3480b7f 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -9,7 +9,7 @@ services: - '8099' # Specify different port to avoid confusion - --scheme - http - image: semitechnologies/weaviate:1.19.6 + image: semitechnologies/weaviate:1.20.2 ports: - 8099:8099 restart: on-failure:0 diff --git a/tests/test_crud.py b/tests/test_crud.py index eb7633ade8..5c6763dfb6 100644 --- a/tests/test_crud.py +++ b/tests/test_crud.py @@ -1,6 +1,5 @@ import pytest import utils -import subprocess @pytest.mark.parametrize( @@ -10,8 +9,9 @@ "./_includes/code/howto/manage-data.cross-refs.py", "./_includes/code/howto/manage-data.import.py", "./_includes/code/howto/manage-data.delete.py", - "./_includes/code/howto/manage-data.classes.py" - + "./_includes/code/howto/manage-data.update.py" + "./_includes/code/howto/manage-data.classes.py", + "./_includes/code/howto/manage-data.multi-tenancy.py", ], ) def test_on_blank_instance(empty_weaviates, script_loc): @@ -26,6 +26,7 @@ def test_on_blank_instance(empty_weaviates, script_loc): @pytest.mark.parametrize( "script_loc", [ + "./_includes/code/howto/manage-data.read.py", "./_includes/code/howto/read-all-objects.py" ], ) @@ -38,40 +39,40 @@ def test_on_edu_demo(empty_weaviates, script_loc): exec(temp_proc_script_loc.read_text()) -@pytest.mark.parametrize( - "script_loc", - [ +# @pytest.mark.parametrize( +# "script_loc", +# [ - ], -) -def test_js(empty_weaviates, script_loc): - temp_proc_script_loc = utils.load_and_prep_temp_file( - script_loc, - lang="js", - custom_replace_pairs=utils.edu_readonly_replacements - ) - try: - # If the script throws an error, this will raise a CalledProcessError - subprocess.check_call(['node', temp_proc_script_loc]) - except subprocess.CalledProcessError as error: - pytest.fail(f'Script {temp_proc_script_loc} failed with error: {error}') +# ], +# ) +# def test_js(empty_weaviates, script_loc): +# temp_proc_script_loc = utils.load_and_prep_temp_file( +# script_loc, +# lang="js", +# custom_replace_pairs=utils.edu_readonly_replacements +# ) +# try: +# # If the script throws an error, this will raise a CalledProcessError +# subprocess.check_call(['node', temp_proc_script_loc]) +# except subprocess.CalledProcessError as error: +# pytest.fail(f'Script {temp_proc_script_loc} failed with error: {error}') -@pytest.mark.parametrize( - "script_loc", - [ - # "./_includes/code/howto/manage-data.cross-refs.ts" # Test currently not working - needs work to fix - ], -) -def test_ts(empty_weaviates, script_loc): - temp_proc_script_loc = utils.load_and_prep_temp_file( - script_loc, - lang="ts", - ) - command = ["node", "--loader=ts-node/esm", temp_proc_script_loc] +# @pytest.mark.parametrize( +# "script_loc", +# [ +# # "./_includes/code/howto/manage-data.cross-refs.ts" # Test currently not working - needs work to fix +# ], +# ) +# def test_ts(empty_weaviates, script_loc): +# temp_proc_script_loc = utils.load_and_prep_temp_file( +# script_loc, +# lang="ts", +# ) +# command = ["node", "--loader=ts-node/esm", temp_proc_script_loc] - try: - # If the script throws an error, this will raise a CalledProcessError - subprocess.check_call(command) - except subprocess.CalledProcessError as error: - pytest.fail(f'Script {temp_proc_script_loc} failed with error: {error}') +# try: +# # If the script throws an error, this will raise a CalledProcessError +# subprocess.check_call(command) +# except subprocess.CalledProcessError as error: +# pytest.fail(f'Script {temp_proc_script_loc} failed with error: {error}')