feat(query): introduce fts_term() function for preparing tokens for F…

…TS5 token matching
pelias · Dec 13, 2021 · 9988ec4 · 9988ec4
1 parent 82db3cd
commit 9988ec4
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 9 deletions.
diff --git a/lib/Queries.js b/lib/Queries.js
@@ -148,16 +148,16 @@ module.exports.matchSubjectObjectGeomIntersects = function( subject, object, cb
   if( 0 > RTREE_THRESHOLD ){ return cb( null, [] ); }
 
   if( isPartialToken ){
-    object = object.replace(/ /g, '_').replace(REMOVE_PARTIAL_TOKEN_REGEX, '');
+    object = object.replace(REMOVE_PARTIAL_TOKEN_REGEX, '');
     if( '' === object.trim() ){ return cb( null, [] ); }
 
     this._queryAll(
       this.prepare( query.match_subject_object_geom_intersects_autocomplete ),
       {
         subject,
         object,
-        subject_quoted: `"${subject}"`,
-        object_quoted: `"${object}"`,
+        subject_fts: fts_term(subject),
+        object_fts: fts_term(object),
         threshold: RTREE_THRESHOLD,
         limit: MAX_RESULTS
       },
@@ -169,12 +169,20 @@ module.exports.matchSubjectObjectGeomIntersects = function( subject, object, cb
       {
         subject,
         object,
-        subject_quoted: `"${subject}"`,
-        object_quoted: `"${object}"`,
+        subject_fts: fts_term(subject),
+        object_fts: fts_term(object),
         threshold: RTREE_THRESHOLD,
         limit: MAX_RESULTS
       },
       cb
     );
   }
 };
+
+// fts_term() converts a regular term (of one or more tokens)
+// into a format which can be used by the FTS5 table.
+//  1. terms should be enclosed in double-quotes
+//  2. spaces should be replaced with underscores (as per the analyzer)
+function fts_term(term) {
+  return `"${term.replace(/ /g, '_')}"`;
+}
diff --git a/query/match_subject_object_geom_intersects.sql b/query/match_subject_object_geom_intersects.sql
@@ -4,7 +4,7 @@ SELECT
 FROM fulltext f1
   JOIN tokens t1 ON (
     f1.rowid = t1.rowid
-    AND f1.fulltext MATCH $subject_quoted
+    AND f1.fulltext MATCH $subject_fts
     AND LIKELY(t1.token = $subject)
   )
     JOIN rtree AS r1 ON t1.id = r1.id
@@ -15,7 +15,7 @@ FROM fulltext f1
         (r1.minY - $threshold) < r2.maxY AND
         (r1.maxY + $threshold) > r2.minY
       )
-        JOIN fulltext AS f2 ON f2.fulltext MATCH $object_quoted
+        JOIN fulltext AS f2 ON f2.fulltext MATCH $object_fts
           JOIN tokens t2 ON (
             f2.rowid = t2.rowid
             AND r2.id = t2.id

diff --git a/query/match_subject_object_geom_intersects_autocomplete.sql b/query/match_subject_object_geom_intersects_autocomplete.sql
@@ -4,7 +4,7 @@ SELECT
 FROM fulltext f1
   JOIN tokens t1 ON (
     f1.rowid = t1.rowid
-    AND f1.fulltext MATCH $subject_quoted
+    AND f1.fulltext MATCH $subject_fts
     AND LIKELY(t1.token = $subject)
   )
     JOIN rtree AS r1 ON t1.id = r1.id
@@ -15,7 +15,7 @@ FROM fulltext f1
         (r1.minY - $threshold) < r2.maxY AND
         (r1.maxY + $threshold) > r2.minY
       )
-        JOIN fulltext AS f2 ON f2.fulltext MATCH $object_quoted OR $object_quoted*
+        JOIN fulltext AS f2 ON f2.fulltext MATCH $object_fts OR $object_fts*
           JOIN tokens t2 ON (
             f2.rowid = t2.rowid
             AND r2.id = t2.id