Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support embedders setting #554

Open
wants to merge 20 commits into
base: main
Choose a base branch
from
Open

Support embedders setting #554

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
333c79a
Added the ability to configure vector-search embeddings in the settings
CommanderStorm Mar 2, 2024
83f543d
Added the ability to configure the `hybrid` keyword in the search
CommanderStorm Apr 16, 2024
a4e50c9
Migrated the testcase to use `_vectors` instead
CommanderStorm Apr 16, 2024
0e044b1
Removed the `experimental-vector-search` feature
CommanderStorm Apr 17, 2024
3a4d876
Merge branch 'main' into vector-search-embedder
curquiza May 14, 2024
a167c50
Merge branch 'main' into vector-search-embedder
CommanderStorm Jun 30, 2024
37b5410
Merge branch 'main' into vector-search-embedder
CommanderStorm Jul 1, 2024
4c7475e
Merge branch 'main' into vector-search-embedder
curquiza Jul 2, 2024
5e72f98
feat: added support for ollama and rest
CommanderStorm Jul 7, 2024
75e5585
chore: improved the documentation
CommanderStorm Jul 7, 2024
464de44
feat: implemnted the `retrieve_vectors` flag
CommanderStorm Jul 8, 2024
4cf2da8
Merge branch 'main' into vector-search-embedder
CommanderStorm Jul 8, 2024
980e714
chore: made sure that `test_hybrid` uses a mocked server
CommanderStorm Jul 8, 2024
cbac495
chore: formatting fixes
CommanderStorm Jul 8, 2024
77399a2
chore: fixed a typo in a doc-comment
CommanderStorm Jul 12, 2024
c69358b
Merge branch 'main' into vector-search-embedder
CommanderStorm Aug 14, 2024
4c94ce5
fix(tests): made the requested changes
CommanderStorm Aug 24, 2024
7398185
fix: changed the rest embedder to the `1.10.0` schema
CommanderStorm Aug 26, 2024
93b0bca
feat: added `headers` support
CommanderStorm Aug 29, 2024
53083ed
Apply suggestions from code review
CommanderStorm Nov 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 4 additions & 11 deletions src/features.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,24 +109,17 @@ mod tests {
use super::*;
use meilisearch_test_macro::meilisearch_test;

#[meilisearch_test]
async fn test_experimental_features_get(client: Client) {
let mut features = ExperimentalFeatures::new(&client);
features.set_vector_store(false);
let _ = features.update().await.unwrap();

let res = features.get().await.unwrap();

assert!(!res.vector_store);
}

/// there is purposely no test which disables this feature to prevent impact on other testcases
/// the setting is shared amongst all indexes
#[meilisearch_test]
async fn test_experimental_features_enable_vector_store(client: Client) {
let mut features = ExperimentalFeatures::new(&client);
features.set_vector_store(true);

let res = features.update().await.unwrap();
assert!(res.vector_store);

let res = features.get().await.unwrap();
assert!(res.vector_store);
}
}
221 changes: 202 additions & 19 deletions src/search.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,22 @@ pub enum Selectors<T> {
All,
}

/// EXPERIMENTAL
#[derive(Debug, Serialize, Clone)]
#[serde(rename_all = "camelCase")]
pub struct HybridSearch<'a> {
/// Indicates one of the embedders configured for the queried index
///
/// **Default: `"default"`**
embedder: &'a str,
/// number between `0` and `1`:
/// - `0.0` indicates full keyword search
/// - `1.0` indicates full semantic search
///
/// **Default: `0.5`**
semantic_ratio: f32,
}

type AttributeToCrop<'a> = (&'a str, Option<usize>);

/// A struct representing a query.
Expand Down Expand Up @@ -346,6 +362,24 @@ pub struct SearchQuery<'a, Http: HttpClient> {

#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) index_uid: Option<&'a str>,

/// EXPERIMENTAL
/// Defines whether to utilise previously defined embedders for semantic searching
#[serde(skip_serializing_if = "Option::is_none")]
pub hybrid: Option<HybridSearch<'a>>,

/// EXPERIMENTAL
/// Defines what vectors an userprovided embedder has gotten for semantic searching
#[serde(skip_serializing_if = "Option::is_none")]
pub vector: Option<&'a [f32]>,

/// EXPERIMENTAL
/// Defines whether vectors for semantic searching are returned in the search results
/// Can Significantly increase the response size.
///
/// **Default: `false`**
#[serde(skip_serializing_if = "Option::is_none")]
retrieve_vectors: Option<bool>,
}

#[allow(missing_docs)]
Expand Down Expand Up @@ -375,6 +409,9 @@ impl<'a, Http: HttpClient> SearchQuery<'a, Http> {
show_ranking_score_details: None,
matching_strategy: None,
index_uid: None,
hybrid: None,
vector: None,
retrieve_vectors: None,
distinct: None,
ranking_score_threshold: None,
}
Expand Down Expand Up @@ -469,6 +506,13 @@ impl<'a, Http: HttpClient> SearchQuery<'a, Http> {
self.filter = Some(Filter::new(Either::Right(filter)));
self
}
pub fn with_retrieve_vectors<'b>(
&'b mut self,
retrieve_vectors: bool,
) -> &'b mut SearchQuery<'a, Http> {
self.retrieve_vectors = Some(retrieve_vectors);
self
}
pub fn with_facets<'b>(
&'b mut self,
facets: Selectors<&'a [&'a str]>,
Expand Down Expand Up @@ -569,6 +613,26 @@ impl<'a, Http: HttpClient> SearchQuery<'a, Http> {
self.index_uid = Some(&self.index.uid);
self
}
/// EXPERIMENTAL
/// Defines whether to utilise previously defined embedders for semantic searching
pub fn with_hybrid<'b>(
&'b mut self,
embedder: &'a str,
semantic_ratio: f32,
) -> &'b mut SearchQuery<'a, Http> {
self.hybrid = Some(HybridSearch {
embedder,
semantic_ratio,
});
self
}
/// EXPERIMENTAL
/// Defines what vectors an userprovided embedder has gotten for semantic searching
pub fn with_vector<'b>(&'b mut self, vector: &'a [f32]) -> &'b mut SearchQuery<'a, Http> {
self.vector = Some(vector);
self
}
#[must_use]
pub fn with_distinct<'b>(&'b mut self, distinct: &'a str) -> &'b mut SearchQuery<'a, Http> {
self.distinct = Some(distinct);
self
Expand Down Expand Up @@ -666,6 +730,36 @@ mod tests {
kind: String,
number: i32,
nested: Nested,
#[serde(default)]
_vectors: Option<Vectors>,
}

#[derive(Debug, Serialize, Deserialize, PartialEq)]
struct Vector {
embeddings: SingleOrMultipleVectors,
regenerate: bool,
}

#[derive(Serialize, Deserialize, Debug, PartialEq)]
#[serde(untagged)]
enum SingleOrMultipleVectors {
Single(Vec<f32>),
Multiple(Vec<Vec<f32>>),
}

#[derive(Debug, Serialize, Deserialize, PartialEq)]
struct Vectors(HashMap<String, Vector>);

impl From<&[f32; 1]> for Vectors {
fn from(value: &[f32; 1]) -> Self {
Vectors(HashMap::from([(
S("default"),
Vector {
embeddings: SingleOrMultipleVectors::Multiple(Vec::from([value.to_vec()])),
regenerate: false,
},
)]))
}
}

impl PartialEq<Map<String, Value>> for Document {
Expand All @@ -678,17 +772,24 @@ mod tests {
}

async fn setup_test_index(client: &Client, index: &Index) -> Result<(), Error> {
// Vector store is enabled for all to have consistent test runs
// This setting is shared by every index
let features = crate::features::ExperimentalFeatures::new(&client)
.set_vector_store(true)
.update()
.await?;
assert_eq!(features.vector_store, true);
let t0 = index.add_documents(&[
Document { id: 0, kind: "text".into(), number: 0, value: S("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."), nested: Nested { child: S("first") } },
Document { id: 1, kind: "text".into(), number: 10, value: S("dolor sit amet, consectetur adipiscing elit"), nested: Nested { child: S("second") } },
Document { id: 2, kind: "title".into(), number: 20, value: S("The Social Network"), nested: Nested { child: S("third") } },
Document { id: 3, kind: "title".into(), number: 30, value: S("Harry Potter and the Sorcerer's Stone"), nested: Nested { child: S("fourth") } },
Document { id: 4, kind: "title".into(), number: 40, value: S("Harry Potter and the Chamber of Secrets"), nested: Nested { child: S("fift") } },
Document { id: 5, kind: "title".into(), number: 50, value: S("Harry Potter and the Prisoner of Azkaban"), nested: Nested { child: S("sixth") } },
Document { id: 6, kind: "title".into(), number: 60, value: S("Harry Potter and the Goblet of Fire"), nested: Nested { child: S("seventh") } },
Document { id: 7, kind: "title".into(), number: 70, value: S("Harry Potter and the Order of the Phoenix"), nested: Nested { child: S("eighth") } },
Document { id: 8, kind: "title".into(), number: 80, value: S("Harry Potter and the Half-Blood Prince"), nested: Nested { child: S("ninth") } },
Document { id: 9, kind: "title".into(), number: 90, value: S("Harry Potter and the Deathly Hallows"), nested: Nested { child: S("tenth") } },
Document { id: 0, kind: "text".into(), number: 0, value: S("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."), nested: Nested { child: S("first") }, _vectors: Some(Vectors::from(&[1000.0]))},
Document { id: 1, kind: "text".into(), number: 10, value: S("dolor sit amet, consectetur adipiscing elit"), nested: Nested { child: S("second") }, _vectors: Some(Vectors::from(&[2000.0])) },
Document { id: 2, kind: "title".into(), number: 20, value: S("The Social Network"), nested: Nested { child: S("third") }, _vectors: Some(Vectors::from(&[3000.0])) },
Document { id: 3, kind: "title".into(), number: 30, value: S("Harry Potter and the Sorcerer's Stone"), nested: Nested { child: S("fourth") }, _vectors: Some(Vectors::from(&[4000.0])) },
Document { id: 4, kind: "title".into(), number: 40, value: S("Harry Potter and the Chamber of Secrets"), nested: Nested { child: S("fift") }, _vectors: Some(Vectors::from(&[5000.0])) },
Document { id: 5, kind: "title".into(), number: 50, value: S("Harry Potter and the Prisoner of Azkaban"), nested: Nested { child: S("sixth") }, _vectors: Some(Vectors::from(&[6000.0])) },
Document { id: 6, kind: "title".into(), number: 60, value: S("Harry Potter and the Goblet of Fire"), nested: Nested { child: S("seventh") }, _vectors: Some(Vectors::from(&[7000.0])) },
Document { id: 7, kind: "title".into(), number: 70, value: S("Harry Potter and the Order of the Phoenix"), nested: Nested { child: S("eighth") }, _vectors: Some(Vectors::from(&[8000.0])) },
Document { id: 8, kind: "title".into(), number: 80, value: S("Harry Potter and the Half-Blood Prince"), nested: Nested { child: S("ninth") }, _vectors: Some(Vectors::from(&[9000.0])) },
Document { id: 9, kind: "title".into(), number: 90, value: S("Harry Potter and the Deathly Hallows"), nested: Nested { child: S("tenth") }, _vectors: Some(Vectors::from(&[10000.0])) },
], None).await?;
let t1 = index
.set_filterable_attributes(["kind", "value", "number"])
Expand Down Expand Up @@ -776,7 +877,8 @@ mod tests {
value: S("dolor sit amet, consectetur adipiscing elit"),
kind: S("text"),
number: 10,
nested: Nested { child: S("second") }
nested: Nested { child: S("second") },
_vectors: Some(Vectors::from(&[2000.0])),
},
&results.hits[0].result
);
Expand Down Expand Up @@ -948,7 +1050,8 @@ mod tests {
value: S("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do…"),
kind: S("text"),
number: 0,
nested: Nested { child: S("first") }
nested: Nested { child: S("first") },
_vectors: None,
},
results.hits[0].formatted_result.as_ref().unwrap()
);
Expand All @@ -963,7 +1066,8 @@ mod tests {
value: S("Lorem ipsum dolor sit amet…"),
kind: S("text"),
number: 0,
nested: Nested { child: S("first") }
nested: Nested { child: S("first") },
_vectors: None,
},
results.hits[0].formatted_result.as_ref().unwrap()
);
Expand All @@ -984,7 +1088,8 @@ mod tests {
value: S("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."),
kind: S("text"),
number: 0,
nested: Nested { child: S("first") }
nested: Nested { child: S("first") },
_vectors: None,
},
results.hits[0].formatted_result.as_ref().unwrap());

Expand All @@ -999,7 +1104,8 @@ mod tests {
value: S("Lorem ipsum dolor sit amet…"),
kind: S("text"),
number: 0,
nested: Nested { child: S("first") }
nested: Nested { child: S("first") },
_vectors: None,
},
results.hits[0].formatted_result.as_ref().unwrap()
);
Expand All @@ -1024,7 +1130,8 @@ mod tests {
value: S("(ꈍᴗꈍ)sed do eiusmod tempor incididunt ut(ꈍᴗꈍ)"),
kind: S("text"),
number: 0,
nested: Nested { child: S("first") }
nested: Nested { child: S("first") },
_vectors: None,
},
results.hits[0].formatted_result.as_ref().unwrap()
);
Expand All @@ -1051,7 +1158,8 @@ mod tests {
value: S("The (⊃｡•́‿•̀｡)⊃ Social ⊂(´• ω •`⊂) Network"),
kind: S("title"),
number: 20,
nested: Nested { child: S("third") }
nested: Nested { child: S("third") },
_vectors: None,
},
results.hits[0].formatted_result.as_ref().unwrap()
);
Expand All @@ -1073,7 +1181,8 @@ mod tests {
value: S("<em>dolor</em> sit amet, consectetur adipiscing elit"),
kind: S("<em>text</em>"),
number: 10,
nested: Nested { child: S("first") }
nested: Nested { child: S("second") },
_vectors: None,
},
results.hits[0].formatted_result.as_ref().unwrap(),
);
Expand All @@ -1088,7 +1197,8 @@ mod tests {
value: S("<em>dolor</em> sit amet, consectetur adipiscing elit"),
kind: S("text"),
number: 10,
nested: Nested { child: S("first") }
nested: Nested { child: S("second") },
_vectors: None,
},
results.hits[0].formatted_result.as_ref().unwrap()
);
Expand Down Expand Up @@ -1274,4 +1384,77 @@ mod tests {

Ok(())
}

/// enable vector searching and configure an userProvided embedder
async fn setup_hybrid_searching(client: &Client, index: &Index) -> Result<(), Error> {
use crate::settings::{Embedder, UserProvidedEmbedderSettings};
let embedder_setting =
Embedder::UserProvided(UserProvidedEmbedderSettings { dimensions: 1 });
let t3 = index
.set_settings(&crate::settings::Settings {
embedders: Some(HashMap::from([("default".to_string(), embedder_setting)])),
..crate::settings::Settings::default()
})
.await?;
t3.wait_for_completion(&client, None, None).await?;
Ok(())
}

#[meilisearch_test]
async fn test_with_vectors(client: Client, index: Index) -> Result<(), Error> {
setup_hybrid_searching(&client, &index).await?;
setup_test_index(&client, &index).await?;

let results: SearchResults<Document> = index
.search()
.with_query("lorem ipsum")
.with_retrieve_vectors(true)
.execute()
.await?;
assert_eq!(results.hits.len(), 1);
let expected = Vectors::from(&[1000.0]);
assert_eq!(results.hits[0].result._vectors, Some(expected));

let results: SearchResults<Document> = index
.search()
.with_query("lorem ipsum")
.with_retrieve_vectors(false)
.execute()
.await?;
assert_eq!(results.hits.len(), 1);
assert_eq!(results.hits[0].result._vectors, None);
Ok(())
}

#[tokio::test]
async fn test_hybrid() -> Result<(), Error> {
// this is mocked as I could not get the hybrid searching to work
// See https://github.com/meilisearch/meilisearch-rust/pull/554 for further context
let mut s = mockito::Server::new_async().await;
let mock_server_url = s.url();
let client = Client::new(mock_server_url, None::<String>)?;
let index = client.index("mocked_index");

let req = r#"{"q":"hello hybrid searching","hybrid":{"embedder":"default","semanticRatio":0.0},"vector":[1000.0]}"#.to_string();
let response = r#"{"hits":[],"offset":null,"limit":null,"estimatedTotalHits":null,"page":null,"hitsPerPage":null,"totalHits":null,"totalPages":null,"facetDistribution":null,"facetStats":null,"processingTimeMs":0,"query":"","indexUid":null}"#.to_string();
let mock_res = s
.mock("POST", "/indexes/mocked_index/search")
.with_status(200)
.match_body(mockito::Matcher::Exact(req))
.with_body(&response)
.expect(1)
.create_async()
.await;
let results: Result<SearchResults<Document>, Error> = index
.search()
.with_query("hello hybrid searching")
.with_hybrid("default", 0.0)
.with_vector(&[1000.0])
.execute()
.await;
mock_res.assert_async().await;
results?; // purposely not done above to have better debugging output

Ok(())
}
}
Loading
Loading