feat: dont deduplicate tokens (#16)

* feat: dont deduplicate tokens * bump version 0.2.5
andylokandy · Apr 19, 2024 · 6a98939 · 6a98939
1 parent 5cd7020
commit 6a98939
Show file tree

Hide file tree

Showing 3 changed files with 2 additions and 4 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -2,7 +2,7 @@
 authors = ["andylokandy <[email protected]>"]
 edition = "2018"
 name = "simsearch"
-version = "0.2.4"
+version = "0.2.5"
 license = "MIT/Apache-2.0"
 
 description = "A simple and lightweight fuzzy search engine that works in memory, searching for similar strings (a pun here)."

diff --git a/src/lib.rs b/src/lib.rs
@@ -148,7 +148,6 @@ where
 
         let mut tokens = self.tokenize(tokens);
         tokens.sort();
-        tokens.dedup();
 
         for token in tokens.clone() {
             self.reverse_map
@@ -214,7 +213,6 @@ where
     pub fn search_tokens(&self, pattern_tokens: &[&str]) -> Vec<Id> {
         let mut pattern_tokens = self.tokenize(pattern_tokens);
         pattern_tokens.sort();
-        pattern_tokens.dedup();
 
         let mut token_scores: HashMap<&str, f64> = HashMap::new();