split sentences also by double newline

eeroel · Jan 18, 2024 · 9795516 · 9795516
1 parent 8d803d5
commit 9795516
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 2 deletions.
diff --git a/hae.cc b/hae.cc
@@ -94,7 +94,7 @@ std::vector<std::string> combine_chunks(std::vector<std::string> &chunks, int64_
 
     std::string buffer = "";
     for (size_t i = 0; i < chunks.size(); ++i) {
-        buffer += chunks[i] + "\n";
+        buffer += chunks[i] + "\n\n";
         // If the chunk has multiple lines, just append it
         if (buffer.length() > min_size) {
             combined.push_back(buffer.substr(0, buffer.length() - 1));
@@ -111,7 +111,9 @@ std::vector<std::string> combine_chunks(std::vector<std::string> &chunks, int64_
 
 std::vector<std::string> split_sentences(const std::string& text) {
     std::string wiki_citation_re = "(\\^\\[[0-9]+\\])*";
-    std::regex full_re(":\\n" + wiki_citation_re + "|[.!?]" + wiki_citation_re + "\\s");
+    std::string double_newline_re = "\r?\n\r?\n";
+
+    std::regex full_re(":\\n" + wiki_citation_re + "|[.!?]" + wiki_citation_re + "\\s" + "|" + double_newline_re);
     size_t prev = 0;
     std::vector<std::string> sentences;
 

diff --git a/test/t3.txt b/test/t3.txt
@@ -1,4 +1,5 @@
 [1m
 This is a test
+
 [0m
 
diff --git a/test/t4.txt b/test/t4.txt
@@ -1,3 +1,4 @@
 [1mThis is a test
+
 [0m
 
diff --git a/test/t5.txt b/test/t5.txt
@@ -3,3 +3,4 @@
 1. [0m	
 
 
+