From bcea5b6cfd4a103662e0c5510f5470d64cf6b210 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Thu, 19 Sep 2024 17:14:25 -0400 Subject: [PATCH] tons of links --- _includes/dynamic-index-async-req.mdx | 4 +- _includes/indexes/dynamic-intro.mdx | 2 +- _includes/indexes/flat-intro.mdx | 2 +- _includes/indexes/hnsw-intro.mdx | 2 +- _includes/pq-compression/overview-text.mdx | 4 +- .../index.mdx | 2 +- .../index.mdx | 2 +- .../index.mdx | 2 +- .../index.mdx | 2 +- .../index.mdx | 2 +- .../index.mdx | 2 +- .../index.mdx | 2 +- .../_core-1-18-include.mdx | 2 +- .../index.mdx | 2 +- .../_core-1-20-include.mdx | 2 +- blog/2023-09-19-pq-rescoring/index.mdx | 2 +- .../_core-1-23-include.mdx | 2 +- .../index.mdx | 2 +- .../_core-1-24-include.mdx | 4 +- .../index.mdx | 2 +- blog/2024-04-02-binary-quantization/index.mdx | 2 +- developers/academy/py/compression/100_pq.mdx | 2 +- developers/academy/py/compression/200_bq.mdx | 3 +- .../academy/py/compression/900_next_steps.mdx | 2 +- .../academy/py/vector_index/100_overview.mdx | 3 +- .../academy/py/vector_index/200_hnsw.mdx | 2 +- .../academy/py/vector_index/220_flat.mdx | 3 +- .../academy/py/vector_index/250_dynamic.mdx | 3 +- .../py/vector_index/900_next_steps.mdx | 2 +- .../concepts/indexing/dynamic-indexes.mdx | 18 +++- .../concepts/indexing/flat-indexes.mdx | 11 ++- .../concepts/indexing/hnsw-indexes.mdx | 96 ++++++++++++++++++- .../concepts/indexing/inverted-indexes.md | 5 - .../concepts/indexing/vector-indexes.md | 94 +----------------- developers/weaviate/concepts/modules.md | 2 +- developers/weaviate/concepts/resources.md | 2 +- developers/weaviate/concepts/storage.md | 2 +- .../config-refs/schema/vector-index.md | 23 ++--- .../compression/bq-compression.md | 6 +- .../compression/pq-compression.md | 2 +- .../compression/sq-compression.md | 4 +- developers/weaviate/introduction.md | 4 +- .../weaviate/manage-data/collections.mdx | 4 +- developers/weaviate/more-resources/faq.md | 4 +- .../more-resources/migration/index.md | 2 +- .../weaviate/more-resources/performance.md | 2 +- .../managing-resources/index.md | 2 +- .../managing-resources/indexing.mdx | 18 ++-- developers/weaviate/starter-guides/schema.md | 4 +- site.redirects.js | 4 +- 50 files changed, 198 insertions(+), 179 deletions(-) diff --git a/_includes/dynamic-index-async-req.mdx b/_includes/dynamic-index-async-req.mdx index c0e1af1629..a6ba6feb4c 100644 --- a/_includes/dynamic-index-async-req.mdx +++ b/_includes/dynamic-index-async-req.mdx @@ -1,3 +1,3 @@ -:::info Dynamic index requires `ASYNC_INDEXING` -To use the `dynamic` vector index type, your Weaviate instance must have the `ASYNC_INDEXING` [environment variable](/developers/weaviate/config-refs/env-vars#general) enabled. This can be done by setting the `ASYNC_INDEXING` environment variable to `true`. For Weaviate Cloud users, this can be enabled through the Weaviate Cloud dashboard. +:::info Dynamic indexes require asynchronous indexing +The `dynamic` vector index type requires asynchronous indexing. To enable asynchronous indexing, set the `ASYNC_INDEXING` [environment variable](/developers/weaviate/config-refs/env-vars#general) to `true`. Weaviate Cloud users can enable asynchronous indexing in the Weaviate Cloud dashboard. ::: diff --git a/_includes/indexes/dynamic-intro.mdx b/_includes/indexes/dynamic-intro.mdx index 8adb6b4b01..04dd2e16a1 100644 --- a/_includes/indexes/dynamic-intro.mdx +++ b/_includes/indexes/dynamic-intro.mdx @@ -1 +1 @@ -[Dynamic index](/developers/weaviate/concepts/vector-index#dynamic-index) offer a flexible approach to indexing. A dynamic index begins as a flat index, and converts automatically to an HNSW index when a [collection](/developers/weaviate/concepts/data#collections) or [tenant](/developers/weaviate/concepts/data#multi-tenancy) reaches a threshold size. \ No newline at end of file +[Dynamic index](/developers/weaviate/concepts/indexing/vector-indexes/dynamic-indexes) offer a flexible approach to indexing. A dynamic index begins as a flat index, and converts automatically to an HNSW index when a [collection](/developers/weaviate/concepts/data#collections) or [tenant](/developers/weaviate/concepts/data#multi-tenancy) reaches a threshold size. \ No newline at end of file diff --git a/_includes/indexes/flat-intro.mdx b/_includes/indexes/flat-intro.mdx index 9881d90b31..231e72aeb2 100644 --- a/_includes/indexes/flat-intro.mdx +++ b/_includes/indexes/flat-intro.mdx @@ -1 +1 @@ -[Flat indexes](/developers/weaviate/concepts/vector-index#flat-index) are memory-efficient. They are disk based indexes that perform brute-force vector searches. Flat indexes work best with small collections, less than 10,000 objects. Vector search times increase linearly with the number of objects. \ No newline at end of file +[Flat indexes](/developers/weaviate/concepts/indexing/vector-indexes/flat-indexes) are memory-efficient. They are disk based indexes that perform brute-force vector searches. Flat indexes work best with small collections, less than 10,000 objects. Vector search times increase linearly with the number of objects. \ No newline at end of file diff --git a/_includes/indexes/hnsw-intro.mdx b/_includes/indexes/hnsw-intro.mdx index 6be8fc9801..9def68b57e 100644 --- a/_includes/indexes/hnsw-intro.mdx +++ b/_includes/indexes/hnsw-intro.mdx @@ -1 +1 @@ -[HNSW indexes](/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) are high-performance, in-memory indexes. HNSW indexes scale well, meaning that vector searches remain fast even for very large data sets. \ No newline at end of file +[HNSW indexes](/developers/weaviate/concepts/indexing/vector-indexes/hnsw-indexes) are high-performance, in-memory indexes. HNSW indexes scale well, meaning that vector searches remain fast even for very large data sets. \ No newline at end of file diff --git a/_includes/pq-compression/overview-text.mdx b/_includes/pq-compression/overview-text.mdx index c82e0dc4f8..aba30f05b8 100644 --- a/_includes/pq-compression/overview-text.mdx +++ b/_includes/pq-compression/overview-text.mdx @@ -1 +1,3 @@ -[Product quantization (PQ)](/developers/weaviate/concepts/vector-index#hnsw-with-product-quantizationpq) is a form of data compression for vectors. PQ reduces the memory footprint of a vector index, so enabling PQ for HNSW lets you work with larger datasets. For a discussion of how PQ saves memory, see [this page](/developers/weaviate/concepts/vector-quantization#product-quantization). +Product quantization (PQ) is a form of data compression for vectors. PQ reduces the memory requirements for vector indexes so you can use larger datasets. + +For a discussion of how PQ saves memory, see [Product quantization](/developers/weaviate/concepts/vector-quantization#product-quantization). diff --git a/blog/2022-07-04-weaviate-1-14-release/index.mdx b/blog/2022-07-04-weaviate-1-14-release/index.mdx index 63b3fbccc9..8349c88d51 100644 --- a/blog/2022-07-04-weaviate-1-14-release/index.mdx +++ b/blog/2022-07-04-weaviate-1-14-release/index.mdx @@ -193,7 +193,7 @@ The REST API CRUD operations now require you to use both an **object ID** and th This ensures that the operations are performed on the correct objects. ### Background -One of Weaviate's features is full CRUD support. CRUD operations enable the mutability of data objects and their vectors, which is a key difference between a vector database and an ANN library. In Weaviate, every data object has an ID (UUID). This ID is stored with the data object in a key-value store. IDs don't have to be globally unique, because in Weaviate [classes](/developers/weaviate/manage-data/collections) act as namespaces. While each class has a different [HNSW index](/developers/weaviate/concepts/vector-index#hnsw), including the store around it, which is isolated on disk. +One of Weaviate's features is full CRUD support. CRUD operations enable the mutability of data objects and their vectors, which is a key difference between a vector database and an ANN library. In Weaviate, every data object has an ID (UUID). This ID is stored with the data object in a key-value store. IDs don't have to be globally unique, because in Weaviate [classes](/developers/weaviate/manage-data/collections) act as namespaces. While each class has a different [HNSW index](/developers/weaviate/concepts/indexing/hnsw-indexes), including the store around it, which is isolated on disk. There was however one point in the API where reusing IDs between classes was causing serious issues. Most noticeable this was for the [v1/objects/{id}](/developers/weaviate/api/rest#tag/objects) REST endpoints. If you wanted to retrieve, modify or delete a data object by its ID, you would just need to specify the ID, without specifying the class name. So if the same ID exists for objects in multiple classes (which is fine because of the namespaces per class), Weaviate would not know which object to address and would address all objects with that ID instead. I.e. if you tried to delete an object by ID, this would result in the deletion of all objects with that ID. diff --git a/blog/2022-08-09-using-cross-encoders-as-reranker-in-multistage-vector-search/index.mdx b/blog/2022-08-09-using-cross-encoders-as-reranker-in-multistage-vector-search/index.mdx index 91af619455..2bf03b1ba2 100644 --- a/blog/2022-08-09-using-cross-encoders-as-reranker-in-multistage-vector-search/index.mdx +++ b/blog/2022-08-09-using-cross-encoders-as-reranker-in-multistage-vector-search/index.mdx @@ -25,7 +25,7 @@ Vector databases like [Weaviate](/developers/weaviate/) use Bi-Encoder machine l The models that compute **dense vector embeddings for data** (which later can be used for **search queries**) are so-called *Bi-Encoder models*. Data vectors and query vectors can be compared by computing the similarity (for example cosine similarity) between vectors. All data items in the database can be represented by a vector embedding using a Bi-Encoder model. -> Note. Vector databases **compute** dense vector embeddings **during import**. Calculating vectors up-front coupled with [ANN algorithms](/developers/weaviate/concepts/vector-index) (for faster retrieval) makes working with Bi-Encoder highly efficient. +> Note. Vector databases **compute** dense vector embeddings **during import**. Calculating vectors up-front coupled with [ANN algorithms](/developers/weaviate/concepts/indexing/vector-indexes) (for faster retrieval) makes working with Bi-Encoder highly efficient. ![Vector Database](./img/vector-database.png) diff --git a/blog/2022-09-13-why-is-vector-search-so-fast/index.mdx b/blog/2022-09-13-why-is-vector-search-so-fast/index.mdx index 1c338f38a7..e48e1d623f 100644 --- a/blog/2022-09-13-why-is-vector-search-so-fast/index.mdx +++ b/blog/2022-09-13-why-is-vector-search-so-fast/index.mdx @@ -98,7 +98,7 @@ Which algorithm works best depends on your project. Performance can be measured So, while ANN is not some magic method that will always find the true k nearest neighbors in a dataset, it can find a pretty good approximation of the true k neighbors. And it can do this in a fraction of the time! ## HNSW in Weaviate -[Weaviate](/developers/weaviate/) is a great example of a vector database that uses ANN algorithms to offer ultra-fast queries. The first ANN algorithm introduced to Weaviate is a custom implementation of [Hierarchical Navigable Small World graphs (HNSW)](/developers/weaviate/concepts/vector-index#hnsw). +[Weaviate](/developers/weaviate/) is a great example of a vector database that uses ANN algorithms to offer ultra-fast queries. The first ANN algorithm introduced to Weaviate is a custom implementation of [Hierarchical Navigable Small World graphs (HNSW)](/developers/weaviate/concepts/indexing/hnsw-indexes). :::tip Read more: -- [Concepts: Vector Indexing # Product Quantization](/developers/weaviate/concepts/vector-index#hnsw-with-product-quantizationpq) +- [Concepts: Vector Indexing # Product Quantization](/developers/weaviate/concepts/indexing/hnsw-indexes) - [How-to configure: Indexes # Vector index (see `pq`)](/developers/weaviate/config-refs/schema/vector-index#weaviates-vector-index) ::: diff --git a/blog/2023-09-19-pq-rescoring/index.mdx b/blog/2023-09-19-pq-rescoring/index.mdx index 2ce62b0a20..e77ee057ad 100644 --- a/blog/2023-09-19-pq-rescoring/index.mdx +++ b/blog/2023-09-19-pq-rescoring/index.mdx @@ -35,7 +35,7 @@ If you’d like to dig deeper into the inner workings of PQ and how these compre In this blog we cover how we can take this one step further, with 1.21 we introduced significant improvements to make PQ even more viable by reducing the recall price you pay in order to save on memory requirements. If you want to learn about these improvements and why using PQ should be a no-brainer now for most users, keep reading! :::info Enabling PQ -To learn how to enable and configure Weaviate to use PQ along with HNSW refer to the docs [here](/developers/weaviate/config-refs/schema/vector-index#how-to-configure-hnsw). You can read more about HNSW+PQ in the documentation [here](/developers/weaviate/concepts/vector-index#hnsw-with-product-quantizationpq). +To learn how to enable and configure Weaviate to use PQ along with HNSW refer to the docs [here](/developers/weaviate/config-refs/schema/vector-index#how-to-configure-hnsw). You can read more about HNSW+PQ in the documentation [here](/developers/weaviate/concepts/indexing/hnsw-indexes). ::: ## The problem with PQ diff --git a/blog/2023-12-19-weaviate-1-23-release/_core-1-23-include.mdx b/blog/2023-12-19-weaviate-1-23-release/_core-1-23-include.mdx index c698bec69e..3f53a87d6a 100644 --- a/blog/2023-12-19-weaviate-1-23-release/_core-1-23-include.mdx +++ b/blog/2023-12-19-weaviate-1-23-release/_core-1-23-include.mdx @@ -56,7 +56,7 @@ We expect that BQ will generally work better for vectors with higher dimensions. When BQ is enabled, a vector cache can be used to improve query performance by storing the quantized vectors of the most recently used data objects. Note that it must be balanced with memory usage considerations. -* Read more about the `flat` index [here](/developers/weaviate/concepts/vector-index#flat-index). +* Read more about the `flat` index [here](/developers/weaviate/concepts/indexing/flat-indexes). ## OSS LLM integration with `generative-anyscale` diff --git a/blog/2024-02-20-grpc-performance-improvements/index.mdx b/blog/2024-02-20-grpc-performance-improvements/index.mdx index ccc60a8279..920d73aa23 100644 --- a/blog/2024-02-20-grpc-performance-improvements/index.mdx +++ b/blog/2024-02-20-grpc-performance-improvements/index.mdx @@ -112,7 +112,7 @@ The chart shows that each query takes 40-70% less time to return results when us Query throughput is usually measured in queries per second (QPS). The chart shows query throughput values based on averages of 10,000 queries.
-The different settings relate to a [vector index (HNSW) parameter](https://weaviate.io/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) `ef`. +The different settings relate to a [vector index (HNSW) parameter](https://weaviate.io/developers/weaviate/concepts/indexing/hnsw-indexes) `ef`. This parameter sets the size of the dynamic candidate list used in the HNSW search algorithm, and drives the speed and accuracy trade-off. In this test, it will impact the ANN-search portion of each query. Regardless, you can see that even at high `ef` numbers the gRPC-driven improvements are significant. diff --git a/blog/2024-02-27-weaviate-1-24-release/_core-1-24-include.mdx b/blog/2024-02-27-weaviate-1-24-release/_core-1-24-include.mdx index 9c63f59dca..0bd37b9cd8 100644 --- a/blog/2024-02-27-weaviate-1-24-release/_core-1-24-include.mdx +++ b/blog/2024-02-27-weaviate-1-24-release/_core-1-24-include.mdx @@ -28,7 +28,7 @@ For details, see [Multiple vectors](/developers/weaviate/config-refs/schema/mult ![HNSW and binary quantization](./img/hnsw-and-bq.png) -Weaviate improves [binary quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) in 1.24 to be faster, more memory efficient, and more cost-effective. Use BQ vector compression with [HNSW indexes](/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) to dramatically improve your query speed. +Weaviate improves [binary quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) in 1.24 to be faster, more memory efficient, and more cost-effective. Use BQ vector compression with [HNSW indexes](/developers/weaviate/concepts/indexing/hnsw-indexes) to dramatically improve your query speed. BQ compresses vector representations while preserving essential information. Uncompressed, Weaviate uses a `float32` to store each dimension. BQ uses one bit per dimension to encode Vector directionality. This means BQ compresses vectors from 32 bits per dimension to 1 bit per dimension - a savings of 32 times the space. This compression significantly reduces storage requirements. @@ -52,7 +52,7 @@ Compare each value in the first vector one with the corresponding value in the s Efficient BQ processing compliments our HNSW index to enable mind dizzyingly fast vector search and throughput! -Let’s talk numbers! You might know from our v1.23 release that combining a [flat index](/developers/weaviate/concepts/vector-index#flat-index) with BQ enables real-time search capabilities. Even so, a brute force a search on 1 million vectors, where each vector has 768 dimensions, takes about 23 milliseconds with a flat index. When you use HNSW and BQ, even with vectors six times as big (4608 dimensions), the same brute force search only takes about 90 milliseconds. +Let’s talk numbers! You might know from our v1.23 release that combining a [flat index](/developers/weaviate/concepts/indexing/flat-indexes) with BQ enables real-time search capabilities. Even so, a brute force a search on 1 million vectors, where each vector has 768 dimensions, takes about 23 milliseconds with a flat index. When you use HNSW and BQ, even with vectors six times as big (4608 dimensions), the same brute force search only takes about 90 milliseconds. A flat index combined with BQ is fast, but compare a HNSW index combined with BQ. Together HNSW and BQ enable vector search capabilities that reach nearly 10,000 queries per second at 85% recall! diff --git a/blog/2024-03-12-enterprise-use-cases-weaviate/index.mdx b/blog/2024-03-12-enterprise-use-cases-weaviate/index.mdx index 49bad256c8..c82e91b110 100644 --- a/blog/2024-03-12-enterprise-use-cases-weaviate/index.mdx +++ b/blog/2024-03-12-enterprise-use-cases-weaviate/index.mdx @@ -54,7 +54,7 @@ These numerical representations are usually presented as an array of numbers and ![embeddings](./img/embeddings.png) -At Weaviate, we’ve implemented a custom [HNSW](https://weaviate.io/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) [indexing](https://weaviate.io/developers/weaviate/concepts/vector-index) algorithm to allow for highly performant querying over your dataset which enables all sorts of business use cases at scale. +At Weaviate, we’ve implemented a custom [HNSW](https://weaviate.io/developers/weaviate/concepts/indexing/hnsw-indexes) [indexing](https://weaviate.io/developers/weaviate/concepts/indexing/vector-indexes) algorithm to allow for highly performant querying over your dataset which enables all sorts of business use cases at scale. ## Features for Enterprise diff --git a/blog/2024-04-02-binary-quantization/index.mdx b/blog/2024-04-02-binary-quantization/index.mdx index bf2b5a516b..06a52a6994 100644 --- a/blog/2024-04-02-binary-quantization/index.mdx +++ b/blog/2024-04-02-binary-quantization/index.mdx @@ -160,7 +160,7 @@ So, what's the bottom line? Can Weaviate deliver lightning-fast brute-force sear There are several advantages to brute force searching your data. Firstly, you can bypass the need for data indexing, saving the time required to build the index. While indexing in Weaviate isn't overly sluggish, brute forcing allows you to skip this step entirely. Secondly, you no longer need to store a proximity graph, resulting in further memory savings. In fact, if you opt to brute force search your data directly from disk, memory usage becomes negligible – a mere 100MB is sufficient to host your application. -Weaviate recently introduced the [flat index](/developers/weaviate/concepts/vector-index#flat-index), offering the option to brute force data either from disk (the default behavior) or by retaining only compressed data in memory and fetching a small selection of full vectors from disk for final candidate rescoring. Both approaches expedite data ingestion compared to the traditional HNSW index while also reducing memory consumption. However, if your requirements demand high performance, HNSW remains the preferred choice. Nevertheless, the flat index presents a cost-effective, high-performing alternative. Furthermore, Weaviate now supports binary quantization (BQ) for use with both the flat and HNSW indexes. +Weaviate recently introduced the [flat index](/developers/weaviate/concepts/indexing/flat-indexes), offering the option to brute force data either from disk (the default behavior) or by retaining only compressed data in memory and fetching a small selection of full vectors from disk for final candidate rescoring. Both approaches expedite data ingestion compared to the traditional HNSW index while also reducing memory consumption. However, if your requirements demand high performance, HNSW remains the preferred choice. Nevertheless, the flat index presents a cost-effective, high-performing alternative. Furthermore, Weaviate now supports binary quantization (BQ) for use with both the flat and HNSW indexes. ### Indexing Time Improvements diff --git a/developers/academy/py/compression/100_pq.mdx b/developers/academy/py/compression/100_pq.mdx index fc37b833f8..691bf97094 100644 --- a/developers/academy/py/compression/100_pq.mdx +++ b/developers/academy/py/compression/100_pq.mdx @@ -8,7 +8,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl import PyCode from '!!raw-loader!./_snippets/100_pq.py'; -[Product quantization](/developers/weaviate/concepts/vector-quantization#product-quantization) (PQ), is a technique used to compress vectors. In Weaviate, it can be used to reduce the size of the in-memory [HNSW index](/developers/weaviate/concepts/vector-index.md#hierarchical-navigable-small-world-hnsw-index), which can improve performance and reduce resource requirements as well as costs. +[Product quantization](/developers/weaviate/concepts/vector-quantization#product-quantization) (PQ), is a technique used to compress vectors. In Weaviate, it can be used to reduce the size of the in-memory [HNSW index](/developers/weaviate/concepts/indexing/hnsw-indexes), which can improve performance and reduce resource requirements as well as costs. ## What is product quantization? diff --git a/developers/academy/py/compression/200_bq.mdx b/developers/academy/py/compression/200_bq.mdx index 87985d0c40..c26c544f2e 100644 --- a/developers/academy/py/compression/200_bq.mdx +++ b/developers/academy/py/compression/200_bq.mdx @@ -8,7 +8,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl import PyCode from '!!raw-loader!./_snippets/200_bq.py'; -[Binary quantization](/developers/weaviate/concepts/vector-quantization#binary-quantization) (BQ), is a technique used to compress vectors. In Weaviate, it can be used to reduce the size of the in-memory [HNSW index](/developers/weaviate/concepts/vector-index.md#hierarchical-navigable-small-world-hnsw-index) or the disk-based [flat index](/developers/weaviate/concepts/vector-index.md#flat-index). +[Binary quantization](/developers/weaviate/concepts/vector-quantization#binary-quantization) (BQ), is a technique used to compress vectors. In Weaviate, it can be used to reduce the size of the in-memory [HNSW index](/developers/weaviate/concepts/indexing/hnsw-indexes) or the disk-based [flat index](/developers/weaviate/concepts/indexing/flat-indexes). For HNSW, BQ can decrease its memory footprint and thus improve performance and reduce resource requirements as well as costs. For the flat index, BQ can reduce the size of the index on disk, which can improve performance. @@ -65,4 +65,3 @@ Some BQ parameters are configurable. An important one is `rescore_limit`, which import DocsFeedback from '/_includes/docs-feedback.mdx'; - diff --git a/developers/academy/py/compression/900_next_steps.mdx b/developers/academy/py/compression/900_next_steps.mdx index b0f61c9370..16718c6857 100644 --- a/developers/academy/py/compression/900_next_steps.mdx +++ b/developers/academy/py/compression/900_next_steps.mdx @@ -10,7 +10,7 @@ There are many more resources available to help you continue your learning journ - [How-to: Configure PQ](/developers/weaviate/configuration/compression/pq-compression.md) - [How-to: Configure BQ](/developers/weaviate/configuration/compression/bq-compression.md) -- [Concepts: vector indexes](/developers/weaviate/concepts/vector-index.md): Vector indexes are critical to how Weaviate works, and compression. Read more here. +- [Concepts: vector indexes](/developers/weaviate/concepts/indexing/vector-indexes): Vector indexes are critical to how Weaviate works, and compression. Read more here. - [Configuration references: Vector index](/developers/weaviate/config-refs/schema/vector-index.md): Configuration references for the vector index, including compression settings. import CTASocials from '../_snippets/cta_socials.mdx'; diff --git a/developers/academy/py/vector_index/100_overview.mdx b/developers/academy/py/vector_index/100_overview.mdx index eff6aed54e..1d0fbd8d25 100644 --- a/developers/academy/py/vector_index/100_overview.mdx +++ b/developers/academy/py/vector_index/100_overview.mdx @@ -16,7 +16,7 @@ A database index is a data structure that organizes data to make searches more e /> -A [vector index](/developers/weaviate/concepts/vector-index.md) is a specialized type of index that is designed to store and search vectors. +A [vector index](/developers/weaviate/concepts/indexing/vector-indexes) is a specialized type of index that is designed to store and search vectors. The choice and configuration of your vector index can significantly impact the performance of your imports as well as searches, and the resource requirements of your Weaviate instance. @@ -92,4 +92,3 @@ We will discuss these in more detail in the following sections. import DocsFeedback from '/_includes/docs-feedback.mdx'; - diff --git a/developers/academy/py/vector_index/200_hnsw.mdx b/developers/academy/py/vector_index/200_hnsw.mdx index 3b7c1c0bc9..3b73318446 100644 --- a/developers/academy/py/vector_index/200_hnsw.mdx +++ b/developers/academy/py/vector_index/200_hnsw.mdx @@ -183,7 +183,7 @@ There are more, advanced HNSW parameters that can be set in Weaviate. These are ## Further resources -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [References: Vector index parameters](/developers/weaviate/config-refs/schema/vector-index.md) - [How-to manage collections](/developers/weaviate/manage-data/collections.mdx) - [Weaviate Academy: Compression](../compression/index.md) diff --git a/developers/academy/py/vector_index/220_flat.mdx b/developers/academy/py/vector_index/220_flat.mdx index ba9fcb984b..a1c2c0da67 100644 --- a/developers/academy/py/vector_index/220_flat.mdx +++ b/developers/academy/py/vector_index/220_flat.mdx @@ -62,7 +62,7 @@ Each of these parameters can be provided when creating a collection in Weaviate. ## Further resources -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [References: Vector index parameters](/developers/weaviate/config-refs/schema/vector-index.md) - [How-to manage collections](/developers/weaviate/manage-data/collections.mdx) - [Weaviate Academy: Compression](../compression/index.md) @@ -72,4 +72,3 @@ Each of these parameters can be provided when creating a collection in Weaviate. import DocsFeedback from '/_includes/docs-feedback.mdx'; - diff --git a/developers/academy/py/vector_index/250_dynamic.mdx b/developers/academy/py/vector_index/250_dynamic.mdx index 8d8ab5f447..34d7dd8b03 100644 --- a/developers/academy/py/vector_index/250_dynamic.mdx +++ b/developers/academy/py/vector_index/250_dynamic.mdx @@ -67,7 +67,7 @@ Additionally, you can specify any of the `flat` and `hnsw` index parameters that ## Further resources -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [References: Vector index parameters](/developers/weaviate/config-refs/schema/vector-index.md) - [How-to manage collections](/developers/weaviate/manage-data/collections.mdx) - [Weaviate Academy: Compression](../compression/index.md) @@ -77,4 +77,3 @@ Additionally, you can specify any of the `flat` and `hnsw` index parameters that import DocsFeedback from '/_includes/docs-feedback.mdx'; - diff --git a/developers/academy/py/vector_index/900_next_steps.mdx b/developers/academy/py/vector_index/900_next_steps.mdx index 72cb5819ba..efdbcf6972 100644 --- a/developers/academy/py/vector_index/900_next_steps.mdx +++ b/developers/academy/py/vector_index/900_next_steps.mdx @@ -18,7 +18,7 @@ Also, in a multi-tenant environment, the "dynamic" index may be a good default c These resources will help you continue your learning journey: -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [References: Vector index parameters](/developers/weaviate/config-refs/schema/vector-index.md) - [How-to manage collections](/developers/weaviate/manage-data/collections.mdx) - [Weaviate Academy: Compression](../compression/index.md) diff --git a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx index ea35eb4471..325107d5d2 100644 --- a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx +++ b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx @@ -3,4 +3,20 @@ title: Dynamic indexed sidebar_position: 40 image: og/docs/indexing.jpg # tags: ['basics']['indexes'] ---- \ No newline at end of file +--- + +:::caution Experimental feature +Available starting in `v1.25`. This is an experimental feature. Use with caution. +::: + +import DynamicAsyncRequirements from '/_includes/dynamic-index-async-req.mdx'; + + + +The flat index is ideal for use cases with a small object count and provides lower memory overhead and good latency. As the object count increases the HNSW index provides a more viable solution as HNSW speeds up search. The goal of the dynamic index is to shorten latencies during querying time at the cost of a larger memory footprint as you scale. + +By configuring a dynamic index, you can automatically switch from flat to HNSW indexes. This switch occurs when the object count exceeds a prespecified threshold (by default 10,000). This functionality only works with async indexing enabled. When the threshold is hit while importing, all the data piles up in the async queue, the HNSW index is constructed in the background and when ready the swap from flat to HNSW is completed. + +Currently, this is only a one-way upgrade from a flat to an HNSW index, it does not support changing back to a flat index even if the object count goes below the threshold due to deletion. + +This is particularly useful in a multi-tenant setup where building an HNSW index per tenant would introduce extra overhead. With a dynamic index, as individual tenants grow their index will switch from flat to HNSW, while smaller tenants' indexes remain flat. diff --git a/developers/weaviate/concepts/indexing/flat-indexes.mdx b/developers/weaviate/concepts/indexing/flat-indexes.mdx index dcfe86f362..68783801be 100644 --- a/developers/weaviate/concepts/indexing/flat-indexes.mdx +++ b/developers/weaviate/concepts/indexing/flat-indexes.mdx @@ -3,4 +3,13 @@ title: Flat indexes sidebar_position: 30 image: og/docs/indexing.jpg # tags: ['basics']['indexes'] ---- \ No newline at end of file +--- + +:::info Added in `v1.23` +::: + +The **flat index** is a simple, lightweight index that is fast to build and has a very small memory footprint. This index type is a good choice for use cases where each end user (i.e. tenant) has their own, isolated, dataset, such as in a SaaS product for example, or a database of isolated record sets. + +As the name suggests, the flat index is a single layer of disk-backed data objects and thus a very small memory footprint. The flat index is a good choice for small collections, such as for multi-tenancy use cases. + +A drawback of the flat index is that it does not scale well to large collections as it has a linear time complexity as a function of the number of data objects, unlike the `hnsw` index which has a logarithmic time complexity. diff --git a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx index 34440b6da1..44ccfccd5a 100644 --- a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx +++ b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx @@ -3,4 +3,98 @@ title: HNSW Indexes sidebar_position: 20 image: og/docs/indexing.jpg # tags: ['basics']['indexes'] ---- \ No newline at end of file +--- + +**Hierarchical Navigable Small World (HNSW)** is an algorithm that works on multi-layered graphs. It is also an index type, and refers to vector indexes that are created using the HNSW algorithm. HNSW indexes enable very fast queries, but rebuilding the index when you add new vectors can be resource intensive. + +Weaviate's `hnsw` index is a custom implementation of the Hierarchical Navigable Small World ([HNSW](https://arxiv.org/abs/1603.09320)) algorithm that offers full [CRUD-support](https://db-engines.com/en/blog_post/87). + + +DWCTODO: implementation https://github.com/weaviate/weaviate/tree/main/adapters/repos/db/vector/hnsw + +At build time, the HNSW algorithm creates a series of layers. At query time, the HNSW algorithm uses the layers to build a list of approximate nearest neighbors (ANN) quickly and efficiently. + +Consider this diagram of a vector search using HNSW. + +![HNSW layers](../img/hnsw-layers.svg "HNSW layers") + +An individual object can exist in more than one layer, but every object in the database is represented in the lowest layer (layer zero in the picture). The layer zero data objects are very well connected to each other. Each layer above the lowest layer has fewer data object, and fewer connections. The data objects in the higher layers correspond to the objects in the lower layers, but each higher layer has exponentially fewer objects than the layer below. The HNSW algorithm takes advantage of the layers to efficiently process large amounts of data. + +When a search query comes in, the HNSW algorithm finds the closest matching data points in the highest layer. Then, HNSW goes one layer deeper, and finds the closest data points in that layer to the ones in the higher layer. These are the nearest neighbors. The algorithm searches the lower layer to create a new list of nearest neighbors. Then, HNSW uses the new list and repeats the process on the next layer down. When it gets to the deepest layer, the HNSW algorithm returns the data objects closest to the search query. + +Since there are relatively few data objects on the higher layers, HNSW has to search fewer objects. This means HNSW 'jumps' over large amounts of data that it doesn't need to search. When a data store has only one layer, the search algorithm can't skip unrelated objects. It has to search significantly more data objects even though they are unlikely to match. + +HNSW is very fast, memory efficient, approach to similarity search. The memory cache only stores the highest layer instead of storing all of the data objects in the lowest layer. When the search moves from a higher layer to a lower one, HNSW only adds the data objects that are closest to the search query. This means HNSW uses a relatively small amount of memory compared to other search algorithms. + +Have another look at the diagram; it demonstrates how the HNSW algorithm searches. The blue search vector in the top layer connects to a partial result in layer one. The objects in layer one lead HNSW to the result set in layer zero. HNSW makes three hops through the layers (the dotted blue lines) and skips objects that are unrelated to the search query. + +If your use case values fast data upload higher than super fast query time and high scalability, then other vector index types may be a better solution (e.g. [Spotify's Annoy](https://github.com/spotify/annoy)). + +### Managing search quality vs speed tradeoffs + +HNSW parameters can be adjusted to adjust search quality against speed. + +The `ef` parameter is a critical setting for balancing the trade-off between search speed and quality. + +The `ef` parameter dictates the size of the dynamic list used by the HNSW algorithm during the search process. A higher `ef` value results in a more extensive search, enhancing accuracy but potentially slowing down the query. + +In contrast, a lower `ef` makes the search faster but might compromise on accuracy. This balance is crucial in scenarios where either speed or accuracy is a priority. For instance, in applications where rapid responses are critical, a lower `ef` might be preferable, even at the expense of some accuracy. Conversely, in analytical or research contexts where precision is paramount, a higher `ef` would be more suitable, despite the increased query time. + +`ef` can be configured explicitly or dynamically. This feature is particularly beneficial in environments with varying query patterns. When `ef` is configured dynamically, Weaviate optimizes the balance between speed and recall based on real-time query requirements. + +To enable dynamic `ef`, set `ef`: -1. Weaviate adjusts the size of the ANN list based on the query response limit. The calculation also takes into account the values of `dynamicEfMin`, `dynamicEfMax`, and `dynamicEfFactor`. + +### Dynamic ef + +The `ef` parameter controls the size of the ANN list at query time. You can configure a specific list size or else let Weaviate configure the list dynamically. If you choose dynamic `ef`, Weaviate provides several options to control the size of the ANN list. + +The length of the list is determined by the query response limit that you set in your query. Weaviate uses the query limit as an anchor and modifies the size of ANN list according to the values you set for the `dynamicEf` parameters. + +- `dynamicEfMin` sets a lower bound on the list length. +- `dynamicEfMax` sets an upper bound on the list length. +- `dynamicEfFactor` sets a range for the list. + +To keep search recall high, the actual dynamic `ef` value stays above `dynamicEfMin` even if the query limit is small enough to suggest a lower value. + +To keep search speed reasonable even when retrieving large result sets, the dynamic `ef` value is limited to `dynamicEfMax`. Weaviate doesn't exceed `dynamicEfMax` even if the query limit is large enough to suggest a higher value. If the query limit is higher than `dynamicEfMax`, `dynamicEfMax` does not have any effect. In this case, dynamic `ef` value is equal to the query limit. + +To determine the length of the ANN list, Weaviate multiples the query limit by `dynamicEfFactor`. The list range is modified by `dynamicEfMin` and `dynamicEfMax`. + +Consider this GraphQL query that sets a limit of 4. + +```graphql +{ + Get { + JeopardyQuestion(limit: 4) { + answer + question + } + } +} +``` + +Imagine the collection has dynamic `ef` configured. + +```json + "vectorIndexConfig": { + "ef": -1, + "dynamicEfMin": 5 + "dynamicEfMax": 25 + "dynamicEfFactor": 10 + } +``` + +The resulting search list has these characteristics. + +- A potential length of 40 objects ( ("dynamicEfFactor": 10) * (limit: 4) ). +- A minimum length of 5 objects ("dynamicEfMin": 5). +- A maximum length of 25 objects ("dynamicEfMax": 25). +- An actual size of 5 to 25 objects. + +If you use the [`docker-compose.yml` file from Weaviate](/developers/weaviate/installation/docker-compose) to run your local instance, the `QUERY_DEFAULTS_LIMIT` environment variable sets a reasonable default query limit. To prevent out of memory errors,`QUERY_DEFAULTS_LIMIT` is significantly lower than `QUERY_MAXIMUM_RESULTS`. + +To change the default limit, edit the value for `QUERY_DEFAULTS_LIMIT` when you configure your Weaviate instance. + +### Deletions + +Cleanup is an async process runs that rebuilds the HNSW graph after deletes and updates. Prior to cleanup, objects are marked as deleted, but they are still connected to the HNSW graph. During cleanup, the edges are reassigned and the objects are deleted for good. diff --git a/developers/weaviate/concepts/indexing/inverted-indexes.md b/developers/weaviate/concepts/indexing/inverted-indexes.md index 219b0c2861..69084943fe 100644 --- a/developers/weaviate/concepts/indexing/inverted-indexes.md +++ b/developers/weaviate/concepts/indexing/inverted-indexes.md @@ -5,7 +5,6 @@ image: og/docs/indexing.jpg # tags: ['basics'] --- - You can configure indexes in Weaviate per collection. Some things to bear in mind: @@ -137,12 +136,8 @@ Some stuff ## Further resources -:::info Related pages - [Concepts: Vector Indexing](./vector-index.md) - [Configuration: Vector index](../config-refs/schema/vector-index.md) -::: - - ## Questions and feedback diff --git a/developers/weaviate/concepts/indexing/vector-indexes.md b/developers/weaviate/concepts/indexing/vector-indexes.md index bad2783344..9aa93d93a3 100644 --- a/developers/weaviate/concepts/indexing/vector-indexes.md +++ b/developers/weaviate/concepts/indexing/vector-indexes.md @@ -46,99 +46,7 @@ You might be also interested in our blog post [Why is vector search to fast?](ht ## Hierarchical Navigable Small World (HNSW) index -**Hierarchical Navigable Small World (HNSW)** is an algorithm that works on multi-layered graphs. It is also an index type, and refers to vector indexes that are created using the HNSW algorithm. HNSW indexes enable very fast queries, but rebuilding the index when you add new vectors can be resource intensive. - -Weaviate's `hnsw` index is a custom implementation of the Hierarchical Navigable Small World ([HNSW](https://arxiv.org/abs/1603.09320)) algorithm that offers full [CRUD-support](https://db-engines.com/en/blog_post/87). - - -DWCTODO: implementation https://github.com/weaviate/weaviate/tree/main/adapters/repos/db/vector/hnsw - -At build time, the HNSW algorithm creates a series of layers. At query time, the HNSW algorithm uses the layers to build a list of approximate nearest neighbors (ANN) quickly and efficiently. - -Consider this diagram of a vector search using HNSW. - -![HNSW layers](../img/hnsw-layers.svg "HNSW layers") - -An individual object can exist in more than one layer, but every object in the database is represented in the lowest layer (layer zero in the picture). The layer zero data objects are very well connected to each other. Each layer above the lowest layer has fewer data object, and fewer connections. The data objects in the higher layers correspond to the objects in the lower layers, but each higher layer has exponentially fewer objects than the layer below. The HNSW algorithm takes advantage of the layers to efficiently process large amounts of data. - -When a search query comes in, the HNSW algorithm finds the closest matching data points in the highest layer. Then, HNSW goes one layer deeper, and finds the closest data points in that layer to the ones in the higher layer. These are the nearest neighbors. The algorithm searches the lower layer to create a new list of nearest neighbors. Then, HNSW uses the new list and repeats the process on the next layer down. When it gets to the deepest layer, the HNSW algorithm returns the data objects closest to the search query. - -Since there are relatively few data objects on the higher layers, HNSW has to search fewer objects. This means HNSW 'jumps' over large amounts of data that it doesn't need to search. When a data store has only one layer, the search algorithm can't skip unrelated objects. It has to search significantly more data objects even though they are unlikely to match. - -HNSW is very fast, memory efficient, approach to similarity search. The memory cache only stores the highest layer instead of storing all of the data objects in the lowest layer. When the search moves from a higher layer to a lower one, HNSW only adds the data objects that are closest to the search query. This means HNSW uses a relatively small amount of memory compared to other search algorithms. - -Have another look at the diagram; it demonstrates how the HNSW algorithm searches. The blue search vector in the top layer connects to a partial result in layer one. The objects in layer one lead HNSW to the result set in layer zero. HNSW makes three hops through the layers (the dotted blue lines) and skips objects that are unrelated to the search query. - -If your use case values fast data upload higher than super fast query time and high scalability, then other vector index types may be a better solution (e.g. [Spotify's Annoy](https://github.com/spotify/annoy)). - -### Managing search quality vs speed tradeoffs - -HNSW parameters can be adjusted to adjust search quality against speed. - -The `ef` parameter is a critical setting for balancing the trade-off between search speed and quality. - -The `ef` parameter dictates the size of the dynamic list used by the HNSW algorithm during the search process. A higher `ef` value results in a more extensive search, enhancing accuracy but potentially slowing down the query. - -In contrast, a lower `ef` makes the search faster but might compromise on accuracy. This balance is crucial in scenarios where either speed or accuracy is a priority. For instance, in applications where rapid responses are critical, a lower `ef` might be preferable, even at the expense of some accuracy. Conversely, in analytical or research contexts where precision is paramount, a higher `ef` would be more suitable, despite the increased query time. - -`ef` can be configured explicitly or dynamically. This feature is particularly beneficial in environments with varying query patterns. When `ef` is configured dynamically, Weaviate optimizes the balance between speed and recall based on real-time query requirements. - -To enable dynamic `ef`, set `ef`: -1. Weaviate adjusts the size of the ANN list based on the query response limit. The calculation also takes into account the values of `dynamicEfMin`, `dynamicEfMax`, and `dynamicEfFactor`. - -### Dynamic ef - -The `ef` parameter controls the size of the ANN list at query time. You can configure a specific list size or else let Weaviate configure the list dynamically. If you choose dynamic `ef`, Weaviate provides several options to control the size of the ANN list. - -The length of the list is determined by the query response limit that you set in your query. Weaviate uses the query limit as an anchor and modifies the size of ANN list according to the values you set for the `dynamicEf` parameters. - -- `dynamicEfMin` sets a lower bound on the list length. -- `dynamicEfMax` sets an upper bound on the list length. -- `dynamicEfFactor` sets a range for the list. - -To keep search recall high, the actual dynamic `ef` value stays above `dynamicEfMin` even if the query limit is small enough to suggest a lower value. - -To keep search speed reasonable even when retrieving large result sets, the dynamic `ef` value is limited to `dynamicEfMax`. Weaviate doesn't exceed `dynamicEfMax` even if the query limit is large enough to suggest a higher value. If the query limit is higher than `dynamicEfMax`, `dynamicEfMax` does not have any effect. In this case, dynamic `ef` value is equal to the query limit. - -To determine the length of the ANN list, Weaviate multiples the query limit by `dynamicEfFactor`. The list range is modified by `dynamicEfMin` and `dynamicEfMax`. - -Consider this GraphQL query that sets a limit of 4. - -```graphql -{ - Get { - JeopardyQuestion(limit: 4) { - answer - question - } - } -} -``` - -Imagine the collection has dynamic `ef` configured. - -```json - "vectorIndexConfig": { - "ef": -1, - "dynamicEfMin": 5 - "dynamicEfMax": 25 - "dynamicEfFactor": 10 - } -``` - -The resulting search list has these characteristics. - -- A potential length of 40 objects ( ("dynamicEfFactor": 10) * (limit: 4) ). -- A minimum length of 5 objects ("dynamicEfMin": 5). -- A maximum length of 25 objects ("dynamicEfMax": 25). -- An actual size of 5 to 25 objects. - -If you use the [`docker-compose.yml` file from Weaviate](/developers/weaviate/installation/docker-compose) to run your local instance, the `QUERY_DEFAULTS_LIMIT` environment variable sets a reasonable default query limit. To prevent out of memory errors,`QUERY_DEFAULTS_LIMIT` is significantly lower than `QUERY_MAXIMUM_RESULTS`. - -To change the default limit, edit the value for `QUERY_DEFAULTS_LIMIT` when you configure your Weaviate instance. - -### Deletions - -Cleanup is an async process runs that rebuilds the HNSW graph after deletes and updates. Prior to cleanup, objects are marked as deleted, but they are still connected to the HNSW graph. During cleanup, the edges are reassigned and the objects are deleted for good. +Add summary ### Asynchronous indexing diff --git a/developers/weaviate/concepts/modules.md b/developers/weaviate/concepts/modules.md index 6992b90f29..ded297c75c 100644 --- a/developers/weaviate/concepts/modules.md +++ b/developers/weaviate/concepts/modules.md @@ -15,7 +15,7 @@ Weaviate has a modularized structure. Functionality such as vectorization or bac The core of Weaviate, without any modules attached, is a pure vector-native database. [![Weaviate modules introduction](./img/weaviate-module-diagram.svg "Weaviate Module Diagram")](./img/weaviate-module-diagram.svg) -Data is stored in Weaviate as the combination of an object and its vector, and these vectors are searchable by the provided [vector index algorithm](../concepts/vector-index.md). Without any vectorizer modules attached, Weaviate does not know how to *vectorize* an object, i.e. *how* to calculate the vectors given an object. +Data is stored in Weaviate as the combination of an object and its vector, and these vectors are searchable by the provided [vector index algorithm](../concepts/indexing/vector-indexes). Without any vectorizer modules attached, Weaviate does not know how to *vectorize* an object, i.e. *how* to calculate the vectors given an object. Depending on the type of data you want to store and search (text, images, etc.), and depending on the use case (like search, question answering, etc., depending on language, classification, ML model, training set, etc.), you can choose and attach a vectorizer module that best fits your use case. Or, you can "bring your own" vectors to Weaviate. diff --git a/developers/weaviate/concepts/resources.md b/developers/weaviate/concepts/resources.md index 84fd76a747..214965fed1 100644 --- a/developers/weaviate/concepts/resources.md +++ b/developers/weaviate/concepts/resources.md @@ -102,7 +102,7 @@ To avoid out-of-memory issues during imports, set `LIMIT_RESOURCES` to `True` or The following tactics can help to reduce Weaviate's memory usage: -- **Use vector compression**. Product quantization (PQ) is a technique that reduces the size of vectors. Vector compression impacts recall performance, so we recommend testing PQ on your dataset before using it in production.

For more information, see [Product Quantization](../concepts/vector-index.md#hnsw-with-product-quantization-pq).
To configure PQ, see [Compression](../configuration/compression/pq-compression.md). +- **Use vector compression**. Product quantization (PQ) is a technique that reduces the size of vectors. Vector compression impacts recall performance, so we recommend testing PQ on your dataset before using it in production.

For more information, see [Product Quantization](../concepts/indexing/hnsw-indexes).
To configure PQ, see [Compression](../configuration/compression/pq-compression.md). - **Reduce the dimensionality of your vectors.** The most effective approach to reducing memory size, is to reduce the number of dimensions per vector. If you have high dimension vectors, consider using a model that uses fewer dimensions. For example, a model that has 384 dimensions uses far less memory than a model with 1536 dimensions. diff --git a/developers/weaviate/concepts/storage.md b/developers/weaviate/concepts/storage.md index 242a764bc2..c0db13ceef 100644 --- a/developers/weaviate/concepts/storage.md +++ b/developers/weaviate/concepts/storage.md @@ -24,7 +24,7 @@ Each shard houses three main components: * An object store, essentially a key-value store * An [inverted index](https://en.wikipedia.org/wiki/Inverted_index) -* A vector index store (plugable, currently a [custom implementation of HNSW](/developers/weaviate/concepts/vector-index.md#hnsw)) +* A vector index store (plugable, currently a [custom implementation of HNSW](/developers/weaviate/concepts/indexing/hnsw-indexes)) #### Object and Inverted Index Store diff --git a/developers/weaviate/config-refs/schema/vector-index.md b/developers/weaviate/config-refs/schema/vector-index.md index 5866b3cc4f..0bd5d97b08 100644 --- a/developers/weaviate/config-refs/schema/vector-index.md +++ b/developers/weaviate/config-refs/schema/vector-index.md @@ -19,16 +19,16 @@ Some HNSW parameters are mutable, but others cannot be modified after you create | :-- | :-- | :-- | :-- | :-- | | `cleanupIntervalSeconds` | integer | 300 | Yes | Cleanup frequency. This value does not normally need to be adjusted. A higher value means cleanup runs less frequently, but it does more in a single batch. A lower value means cleanup is more frequent, but it may be less efficient on each run. | | `distance` | string | `cosine` | No | Distance metric. The metric that measures the distance between two arbitrary vectors. For available distance metrics, see [supported distance metrics](/developers/weaviate/config-refs/distances.md). | -| `ef` | integer | -1 | Yes | Balance search speed and recall. `ef` is the size of the dynamic list that the HNSW uses during search. Search is more accurate when `ef` is higher, but it is also slower. `ef` values greater than 512 show diminishing improvements in recall.

Dynamic `ef`. Weaviate automatically adjusts the `ef` value and creates a dynamic `ef` list when `ef` is set to -1. For more details, see [dynamic ef](../../concepts/vector-index.md#dynamic-ef). | +| `ef` | integer | -1 | Yes | Balance search speed and recall. `ef` is the size of the dynamic list that the HNSW uses during search. Search is more accurate when `ef` is higher, but it is also slower. `ef` values greater than 512 show diminishing improvements in recall.

Dynamic `ef`. Weaviate automatically adjusts the `ef` value and creates a dynamic `ef` list when `ef` is set to -1. For more details, see [dynamic ef](../../concepts/indexing/hnsw-indexes#dynamic-ef). | | `efConstruction` | integer | 128 | No | Balance index search speed and build speed. A high `efConstruction` value means you can lower your `ef` settings, but importing is slower.

`efConstruction` must be greater than 0. | | `maxConnections` | integer | 32 | No | Maximum number of connections per element. `maxConnections` is the connection limit per layer for layers above the zero layer. The zero layer can have (2 * maxConnections) connections.

`maxConnections` must be greater than 0. | -| `dynamicEfMin` | integer | 100 | Yes | *New in `v1.10.0`.*

Lower bound for [dynamic `ef`](../../concepts/vector-index.md#dynamic-ef). Protects against a creating search list that is too short.

This setting is only used when `ef` is -1. | -| `dynamicEfMax` | integer | 500 | Yes | *New in `v1.10.0`.*

Upper bound for [dynamic `ef`](../../concepts/vector-index.md#dynamic-ef). Protects against creating a search list that is too long.

If `dynamicEfMax` is higher than the limit, `dynamicEfMax` does not have any effect. In this case, `ef` is the limit.

This setting is only used when `ef` is -1. | -| `dynamicEfFactor` | integer | 8 | Yes | *New in `v1.10.0`.*

Multiplier for [dynamic `ef`](../../concepts/vector-index.md#dynamic-ef). Sets the potential length of the search list.

This setting is only used when `ef` is -1. | +| `dynamicEfMin` | integer | 100 | Yes | *New in `v1.10.0`.*

Lower bound for [dynamic `ef`](../../concepts/indexing/hnsw-indexes#dynamic-ef). Protects against a creating search list that is too short.

This setting is only used when `ef` is -1. | +| `dynamicEfMax` | integer | 500 | Yes | *New in `v1.10.0`.*

Upper bound for [dynamic `ef`](../../concepts/indexing/hnsw-indexes#dynamic-ef). Protects against creating a search list that is too long.

If `dynamicEfMax` is higher than the limit, `dynamicEfMax` does not have any effect. In this case, `ef` is the limit.

This setting is only used when `ef` is -1. | +| `dynamicEfFactor` | integer | 8 | Yes | *New in `v1.10.0`.*

Multiplier for [dynamic `ef`](../../concepts/indexing/hnsw-indexes#dynamic-ef). Sets the potential length of the search list.

This setting is only used when `ef` is -1. | | `flatSearchCutoff` | integer | 40000 | Yes | Optional. Threshold for the [flat-search cutoff](/developers/weaviate/concepts/prefiltering.md#flat-search-cutoff). To force a vector index search, set `"flatSearchCutoff": 0`. | -| `skip` | boolean | `false` | No | When true, do not index the collection.

Weaviate decouples vector creation and vector storage. If you skip vector indexing, but a vectorizer is configured (or a vector is provided manually), Weaviate logs a warning each import.

To skip indexing and vector generation, set `"vectorizer": "none"` when you set `"skip": true`.

See [When to skip indexing](../../concepts/vector-index.md#when-to-skip-indexing). | -| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](../../concepts/vector-index.md#vector-cache-considerations). | -| `pq` | object | -- | Yes | Enable and configure [product quantization (PQ)](/developers/weaviate/concepts/vector-index.md#hnsw-with-product-quantizationpq) compression.

PQ assumes some data has already been loaded. You should have 10,000 to 100,000 vectors per shard loaded before you enable PQ.

For PQ configuration details, see [PQ configuration parameters](#pq-configuration-parameters). | +| `skip` | boolean | `false` | No | When true, do not index the collection.

Weaviate decouples vector creation and vector storage. If you skip vector indexing, but a vectorizer is configured (or a vector is provided manually), Weaviate logs a warning each import.

To skip indexing and vector generation, set `"vectorizer": "none"` when you set `"skip": true`.

See [When to skip indexing](../../concepts/indexing/hnsw-indexes#when-to-skip-indexing). | +| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](../../concepts/indexing/hnsw-indexes). | +| `pq` | object | -- | Yes | Enable and configure [product quantization (PQ)](/developers/weaviate/indexing/hnsw-indexes) compression.

PQ assumes some data has already been loaded. You should have 10,000 to 100,000 vectors per shard loaded before you enable PQ.

For PQ configuration details, see [PQ configuration parameters](#pq-configuration-parameters). | ### Database parameters for HNSW @@ -140,7 +140,7 @@ Flat indexes are recommended for use cases where the number of objects per index | Parameter | Type | Default | Changeable | Details | | :-- | :-- | :-- | :-- | :-- | -| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](../../concepts/vector-index.md#vector-cache-considerations). | +| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](../../concepts/indexing/vector-indexes). | | `bq` | object | -- | No | Enable and configure [binary quantization (BQ)](../../concepts/vector-quantization.md#binary-quantization) compression.

For BQ configuration details, see [BQ configuration parameters](#bq-configuration-parameters). | ### BQ configuration parameters @@ -194,10 +194,11 @@ Use these parameters to configure the index type and their properties. They can
How to select the index type -Generally, the `hnsw` index type is recommended for most use cases. The `flat` index type is recommended for use cases where the data the number of objects per index is low, such as in multi-tenancy cases. You can also opt for the `dynamic` index which will initially configure a `flat` index and once the object count exceeds a specified threshold it will automatically convert to an `hnsw` index. +The `hnsw` index type is recommended for most use cases. -See [this section](../../concepts/vector-index.md#which-vector-index-is-right-for-me) for more information about the different index types and how to choose between them. +The `flat` index type is recommended for use cases where the number of objects per index is low, such as in multi-tenancy cases. +The `dynamic` index is useful if your collection size starts small and grows. A dynamic index starts as a flat index and converts to an hnsw index when the object count exceeds a specified threshold.
If faster import speeds are desired, [asynchronous indexing](#asynchronous-indexing) allows de-coupling of indexing from object creation. @@ -303,7 +304,7 @@ import MultiVectorSupport from '/_includes/multi-vector-support.mdx'; ## Related pages - [Concepts: Indexing](../../concepts/indexing.md) -- [Concepts: Vector Indexing](../../concepts/vector-index.md) +- [Concepts: Vector Indexing](../../concepts/indexing/vector-indexes) ## Questions and feedback diff --git a/developers/weaviate/configuration/compression/bq-compression.md b/developers/weaviate/configuration/compression/bq-compression.md index f48c7621bc..e86fef8a88 100644 --- a/developers/weaviate/configuration/compression/bq-compression.md +++ b/developers/weaviate/configuration/compression/bq-compression.md @@ -17,7 +17,7 @@ import GoCode from '!!raw-loader!/_includes/code/howto/configure.bq-compression. import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/bq-compression.java'; :::info Added in `v1.23` -BQ is available for the [`flat` index](/developers/weaviate/concepts/vector-index.md#flat-index) type from `v1.23` onwards and for the [`hnsw` index](/developers/weaviate/concepts/vector-index.md#hnsw-index) type from `v1.24`. +BQ is available for the [`flat` index](/developers/weaviate/concepts/indexing/flat-indexes) type from `v1.23` onwards and for the [`hnsw` index](/developers/weaviate/concepts/indexing/hnsw-indexes) type from `v1.24`. ::: Binary quantization (BQ) is a vector compression technique that can reduce the size of a vector. @@ -104,7 +104,7 @@ The following parameters are available for BQ compression, under `vectorIndexCon | `bq` : `enabled` | boolean | `false` | Enable BQ. Weaviate uses binary quantization (BQ) compression when `true`.

The Python client v4 does not use the `enabled` parameter. To enable BQ with the v4 client, set a `quantizer` in the collection definition. | | `bq` : `rescoreLimit` | integer | -1 | The minimum number of candidates to fetch before rescoring. | | `bq` : `cache` | boolean | `false` | Whether to use the vector cache. | -| `vectorCacheMaxObjects` | integer | `1e12` | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/vector-index.md#vector-cache-considerations). | +| `vectorCacheMaxObjects` | integer | `1e12` | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/indexing/vector-indexes). | For example: @@ -173,7 +173,7 @@ import MultiVectorCompress from '/_includes/multi-vector-compress.mdx'; ## Related pages - [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [Concepts: Vector quantization](/developers/weaviate/concepts/vector-quantization.md) - [Tutorial: Schema](/developers/weaviate/starter-guides/schema) diff --git a/developers/weaviate/configuration/compression/pq-compression.md b/developers/weaviate/configuration/compression/pq-compression.md index 7b76bb99d6..0d2313b39d 100644 --- a/developers/weaviate/configuration/compression/pq-compression.md +++ b/developers/weaviate/configuration/compression/pq-compression.md @@ -375,7 +375,7 @@ import MultiVectorCompress from '/_includes/multi-vector-compress.mdx'; ## Related pages - [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [Concepts: Vector quantization](/developers/weaviate/concepts/vector-quantization.md) - [Guide: Schemas and collection definitions](/developers/weaviate/starter-guides/schema) diff --git a/developers/weaviate/configuration/compression/sq-compression.md b/developers/weaviate/configuration/compression/sq-compression.md index 985d4be8c7..41f40fd01f 100644 --- a/developers/weaviate/configuration/compression/sq-compression.md +++ b/developers/weaviate/configuration/compression/sq-compression.md @@ -60,7 +60,7 @@ To tune SQ, set these `vectorIndexConfig` parameters. | `sq`: `rescoreLimit` | integer | -1 | The minimum number of candidates to fetch before rescoring. | | `sq`: `trainingLimit` | integer | 100000 | The size of the training set to determine scalar bucket boundaries. | | `sq`: `cache` | boolean | `false` | Use the vector cache when true. | -| `vectorCacheMaxObjects` | integer | `1e12` | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/vector-index.md#vector-cache-considerations). | +| `vectorCacheMaxObjects` | integer | `1e12` | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/indexing/vector-indexes). | @@ -91,7 +91,7 @@ import MultiVectorCompress from '/_includes/multi-vector-compress.mdx'; ## Related pages - [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [Concepts: Vector quantization](/developers/weaviate/concepts/vector-quantization.md) - [Tutorial: Schema](/developers/weaviate/starter-guides/schema) diff --git a/developers/weaviate/introduction.md b/developers/weaviate/introduction.md index 5935218f42..87f639b80f 100644 --- a/developers/weaviate/introduction.md +++ b/developers/weaviate/introduction.md @@ -128,7 +128,7 @@ Weaviate is an open source vector database that stores both objects and vectors. **Weaviate in a nutshell**: * Weaviate is an open source [vector database](https://weaviate.io/blog/what-is-a-vector-database). -* Weaviate allows you to store and retrieve data objects based on their semantic properties by indexing them with [vectors](./concepts/vector-index.md). +* Weaviate allows you to store and retrieve data objects based on their semantic properties by indexing them with [vectors](./concepts/indexing/vector-indexes). * Weaviate can be used stand-alone (aka _bring your vectors_) or with a variety of [modules](./modules/index.md) that can do the vectorization for you and extend the core capabilities. * Weaviate has a [GraphQL-API](./api/graphql/index.md) to access your data easily. * Weaviate is fast (check our [open source benchmarks](./benchmarks/index.md)). @@ -186,7 +186,7 @@ Within Weaviate, all individual data objects are based on a class property struc You can add data to Weaviate through the [RESTful API](/developers/weaviate/api/rest) end-points and retrieve data through the [GraphQL interface](./api/graphql/index.md). -Weaviate's [vector indexing mechanism is modular](./concepts/vector-index.md), and the current available plugin is the Hierarchical Navigable Small World (HNSW) multilayered graph. +Weaviate's [vector indexing mechanism is modular](./concepts/indexing/vector-indexes), and the current available plugin is the Hierarchical Navigable Small World (HNSW) multilayered graph. ## What are Weaviate modules? diff --git a/developers/weaviate/manage-data/collections.mdx b/developers/weaviate/manage-data/collections.mdx index 0b171dd6d3..b63d7f3876 100644 --- a/developers/weaviate/manage-data/collections.mdx +++ b/developers/weaviate/manage-data/collections.mdx @@ -382,7 +382,7 @@ The vector index type can be set for each collection at creation time, between `
Additional information -- Read more about index types & compression in [Concepts: Vector index](../concepts/vector-index.md). +- Read more about index types & compression in [Concepts: Vector index](../concepts/indexing/vector-indexes).
@@ -440,7 +440,7 @@ Various vector index parameters are configurable at collection creation time, in
Additional information -- Read more about index types & compression in [Concepts: Vector index](../concepts/vector-index.md). +- Read more about index types & compression in [Concepts: Vector index](../concepts/indexing/vector-indexes).
diff --git a/developers/weaviate/more-resources/faq.md b/developers/weaviate/more-resources/faq.md index 1ab6f7cb1f..1869163a08 100644 --- a/developers/weaviate/more-resources/faq.md +++ b/developers/weaviate/more-resources/faq.md @@ -338,9 +338,9 @@ If you need a higher search quality for a given limit you can consider the follo > More information: > > - [Weaviate, an ANN Database with CRUD support – DB-Engines.com](https://db-engines.com/en/blog_post/87) ⬅️ best resource on the topic -> - [Weaviate's HNSW implementation in the docs](/developers/weaviate/concepts/vector-index.md#hnsw) +> - [Weaviate's HNSW implementation in the docs](/developers/weaviate/concepts/indexing/hnsw-indexes) > -> _Note I: HNSW is just one implementation in Weaviate, but Weaviate can support multiple indexing algoritmns as outlined [here](/developers/weaviate/concepts/vector-index.md)_ +> _Note I: HNSW is just one implementation in Weaviate, but Weaviate can support multiple indexing algoritmns as outlined [here](/developers/weaviate/concepts/indexing/vector-indexes)_ diff --git a/developers/weaviate/more-resources/migration/index.md b/developers/weaviate/more-resources/migration/index.md index 6ad50decab..548c74118b 100644 --- a/developers/weaviate/more-resources/migration/index.md +++ b/developers/weaviate/more-resources/migration/index.md @@ -624,7 +624,7 @@ With the modularization, it becomes possible to vectorize non-text objects. Sear * The `vectorizer` indicates which module (if any) are responsible for vectorization. * The `moduleConfig` allows configuration per module (by name). * See [here](#text2vec-contextionary) for Contextionary specific property configuration. - * The `vectorIndexType` allows the choosing the vector index (defaults to [HNSW](/developers/weaviate/concepts/vector-index.md#hnsw)) + * The `vectorIndexType` allows the choosing the vector index (defaults to [HNSW](/developers/weaviate/concepts/indexing/hnsw-indexes)) * The `vectorIndexConfig` is an arbitrary object passed to the index for config (defaults can be found [here](/developers/weaviate/config-refs/schema/vector-index.md#how-to-configure-hnsw) ) All changes are in this example: diff --git a/developers/weaviate/more-resources/performance.md b/developers/weaviate/more-resources/performance.md index ea2e17fad2..89c6a30055 100644 --- a/developers/weaviate/more-resources/performance.md +++ b/developers/weaviate/more-resources/performance.md @@ -28,7 +28,7 @@ The inverted index currently does not do any weighing (e.g. tf-idf) for sorting, ## Vector index Everything that has a vector, thus every data object in Weaviate, is also indexed in the vector index. Weaviate currently supports [HNSW](https://arxiv.org/abs/1603.09320) and flat vector indexes. -See [Concepts: vector index](../concepts/vector-index.md) for more information about the vector index. +See [Concepts: vector index](../concepts/indexing/vector-indexes) for more information about the vector index. ## Costs of queries and operations diff --git a/developers/weaviate/starter-guides/managing-resources/index.md b/developers/weaviate/starter-guides/managing-resources/index.md index bba5048aea..59b47a6ca6 100644 --- a/developers/weaviate/starter-guides/managing-resources/index.md +++ b/developers/weaviate/starter-guides/managing-resources/index.md @@ -168,7 +168,7 @@ Consider a strategy of deactivating tenants that are not frequently accessed, an - [Starter guide: Compression](./compression.mdx) - [Starter guide: Indexing](./indexing.mdx) - [Starter guide: Tenant states](./tenant-states.mdx) -- [Concepts: Vector Index](../../concepts/vector-index.md) +- [Concepts: Vector Index](../../concepts/indexing/vector-indexes) - [Concepts: Vector Quantization](../../concepts/vector-quantization.md) - [Concepts: Multi-Tenancy](../../concepts/data.md#multi-tenancy) - [How-to: Set the vector index type](../../manage-data/collections.mdx#set-vector-index-type) diff --git a/developers/weaviate/starter-guides/managing-resources/indexing.mdx b/developers/weaviate/starter-guides/managing-resources/indexing.mdx index 826cb238a9..607d3494ae 100644 --- a/developers/weaviate/starter-guides/managing-resources/indexing.mdx +++ b/developers/weaviate/starter-guides/managing-resources/indexing.mdx @@ -65,19 +65,21 @@ As a result, flat indexes are best suited for cases where the number of objects :::info Added in `v1.25` ::: -import DynamicAsyncRequirements from '/_includes/dynamic-index-async-req.mdx'; - - - import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; This can be particularly useful in multi-tenant configurations, where different tenants may have different numbers of objects. With a dynamic index, you can avoid the overhead of an HNSW index when it's not needed. -The threshold size is 10,000 objects by default. You can configure the threshold size when you create the dynamic index. +The `dynamic` vector index type requires asynchronous indexing. To enable asynchronous indexing, set the `ASYNC_INDEXING` [environment variable](/developers/weaviate/config-refs/env-vars#general) to `true`. + +Configure the settings for the flat and HNSW indexes when you create the collection definition. The dynamic index uses the flat index configuration when the collection is small. It converts the flat index to an HNSW index when the collection size reaches the dynamic index threshold. + +The threshold size is 10,000 objects by default. Configure the threshold size when you create the dynamic index. -This table shows how a dynamic index changes as the number of objects in a collection grows. The assumed set up is a dynamic index with: +##### Dynamic index behavior + +This table shows how a dynamic index changes as the number of objects in a collection grows. The table assumes a dynamic index configure like this: - A threshold of 10,000 objects. - Flat index + BQ configuration. @@ -90,10 +92,6 @@ This table shows how a dynamic index changes as the number of objects in a colle | 100,000 | HNSW | Training | The collection object count == PQ/SQ training threshold. | | 100,001 | HNSW | PQ/SQ | PQ/SQ is active. | -:::info Dynamic index requires flat and HNSW index settings -A dynamic index requires its flat and HNSW index settings at creation time. The dynamic index will use the flat index settings initially, then automatically switch to the HNSW index with provided settings when the threshold is reached. -::: - ### Asynchronous vector indexing :::info Added in `v1.22` diff --git a/developers/weaviate/starter-guides/schema.md b/developers/weaviate/starter-guides/schema.md index a6dd0d96e4..e5e30451cc 100644 --- a/developers/weaviate/starter-guides/schema.md +++ b/developers/weaviate/starter-guides/schema.md @@ -211,9 +211,9 @@ import SchemaWithMT from '/_includes/code/tutorial.schema.multi-tenancy.mdx'; ### Index settings -Weaviate uses two types of indexes: [vector indexes](../concepts/vector-index.md) and [inverted indexes](../concepts/indexing.md#inverted-indexes). Vector indexes are used to store and organize vectors for fast vector similarity-based searches. Inverted indexes are used to store data for fast filtering and keyword searches. +Weaviate uses two types of indexes: [vector indexes](../concepts/indexing/vector-indexes) and [inverted indexes](../concepts/indexing.md#inverted-indexes). Vector indexes are used to store and organize vectors for fast vector similarity-based searches. Inverted indexes are used to store data for fast filtering and keyword searches. -The default vector index type is [HNSW](../concepts/vector-index.md#hierarchical-navigable-small-world-hnsw-index). The other options are [flat](../concepts/vector-index.md#flat-index), which is suitable for small collections, such as those in a multi-tenancy collection, or [dynamic](../concepts/vector-index.md#dynamic-index), which starts as a flat index before switching to an HNSW index if its size grows beyond a predetermined threshold. +The default vector index type is [HNSW](../concepts/indexing/hnsw-indexes). The other options are [flat](../concepts/indexing/flat-indexes), which is suitable for small collections, such as those in a multi-tenancy collection, or [dynamic](../concepts/indexing/dynamic-indexes), which starts as a flat index before switching to an HNSW index if its size grows beyond a predetermined threshold. import SchemaWithIndexSettings from '/_includes/code/tutorial.schema.index-settings.mdx'; diff --git a/site.redirects.js b/site.redirects.js index 61a530528c..6c5745b48d 100644 --- a/site.redirects.js +++ b/site.redirects.js @@ -19,7 +19,7 @@ const siteRedirects = { from: '/developers/weaviate/current/core-knowledge/basics', }, { - to: '/developers/weaviate/concepts/vector-index', + to: '/developers/weaviate/concepts/indexing/vector-indexes', from: [ '/developers/weaviate/current/vector-index-plugins', '/developers/weaviate/current/vector-index-plugins/hnsw', @@ -219,7 +219,7 @@ const siteRedirects = { // Remove BPR page { - to: '/developers/weaviate/concepts/vector-index', + to: '/developers/weaviate/concepts/indexing/vector-indexes', from: '/developers/weaviate/concepts/binary-passage-retrieval', },