sparse: make bm25 avgdl support value smaller than 1 (#1077)

Signed-off-by: Shawn Wang <[email protected]>
zilliztech · Feb 15, 2025 · 385964a · 385964a
1 parent 8106b1c
commit 385964a
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 1 deletion.
diff --git a/include/knowhere/config.h b/include/knowhere/config.h
@@ -752,7 +752,7 @@ class BaseConfig : public Config {
         // type is WAND.
         KNOWHERE_CONFIG_DECLARE_FIELD(bm25_avgdl)
             .allow_empty_without_default()
-            .set_range(1, std::numeric_limits<CFG_FLOAT::value_type>::max())
+            .set_range(0, std::numeric_limits<CFG_FLOAT::value_type>::max())
             .description("average document length")
             .for_train_and_search()
             .for_iterator()

diff --git a/src/common/comp/brute_force.cc b/src/common/comp/brute_force.cc
@@ -56,6 +56,7 @@ GetDocValueComputer(const BruteForceConfig& cfg) {
         auto k1 = cfg.bm25_k1.value();
         auto b = cfg.bm25_b.value();
         auto avgdl = cfg.bm25_avgdl.value();
+        avgdl = std::max(avgdl, 1.0f);
         return sparse::GetDocValueBM25Computer<T>(k1, b, avgdl);
     } else {
         return expected<sparse::DocValueComputer<T>>::Err(Status::invalid_metric_type,

diff --git a/src/index/sparse/sparse_index_node.cc b/src/index/sparse/sparse_index_node.cc
@@ -371,6 +371,9 @@ class SparseInvertedIndexNode : public IndexNode {
             auto k1 = cfg.bm25_k1.value();
             auto b = cfg.bm25_b.value();
             auto avgdl = cfg.bm25_avgdl.value();
+            // avgdl is used as a denominator in BM25 score computation,
+            // so it should be at least 1.0 to avoid division by zero.
+            avgdl = std::max(avgdl, 1.0f);
 
             if (use_wand || cfg.inverted_index_algo.value() == "DAAT_WAND") {
                 auto index = new sparse::InvertedIndex<T, uint16_t, sparse::InvertedIndexAlgo::DAAT_WAND, mmapped>(

diff --git a/src/index/sparse/sparse_inverted_index.h b/src/index/sparse/sparse_inverted_index.h
@@ -145,6 +145,7 @@ class InvertedIndex : public BaseInvertedIndex<DType> {
                                                           "avgdl must be supplied during searching");
         }
         auto avgdl = cfg.bm25_avgdl.value();
+        avgdl = std::max(avgdl, 1.0f);
         if constexpr (algo == InvertedIndexAlgo::DAAT_WAND || algo == InvertedIndexAlgo::DAAT_MAXSCORE) {
             // daat_wand and daat_maxscore: search time k1/b must equal load time config.
             if ((cfg.bm25_k1.has_value() && cfg.bm25_k1.value() != bm25_params_->k1) ||