Merge pull request #5 from Terminal-Systems/feat/similar-filter

Add similar_to filter

README.md

@@ -52,26 +52,30 @@ Current available filters are:
   <columnName>_gte: Greater than or equal
   <columnName>_lte: Less than or equal
   <columnName>_not_in: Not in (Receives an array of values)
-  <columnName>_contains: String contains (uses full-text-search)
-  <columnName>_not_contains: String not contains (uses full-text-search)
+  <columnName>_contains: String contains
+  <columnName>_not_contains: String not contains
   <columnName>_starts_with: String starts with
   <columnName>_not_starts_with: String does not start with
   <columnName>_ends_with: String ends with
   <columnName>_not_ends_with: String does not end with
+  <columnName>_similar_to: String is similar to
 ```
 
 ### Note
 
-Indexes can greatly increase the performance of filters. You should consider adding indexes for the filterable columns. A normal index should be enough, but for full-text-search filters you can add a GIN or GiST index like in the following example:
+`similar_to` filter requires Postgres `pg_trgm` extension. A migration that creates the extension and creates a GIN index for faster "similar to" queries can look like this:
 
 ```javascript
 exports.up = async (knex) => {
-  await knex.raw('CREATE INDEX posts_title_fts_index ON posts USING gin(tsvector(title))');
+  await knex.raw('CREATE EXTENSION pg_trgm');
+  await knex.raw('CREATE INDEX posts_title_trgm_index ON posts USING gin(title gin_trgm_ops)');
 };
 
 exports.down = async (knex) => {
-  await knex.raw('DROP INDEX posts_title_fts_index');
+  await knex.raw('DROP EXTENSION pg_trgm');
+  await knex.raw('DROP INDEX posts_title_trgm_index');
 };
+
 ```
 
 ## Options
@@ -157,6 +161,10 @@ const query = knexFlexFilter(
 );
 ```
 
+### caseInsensitiveSearch
+
+Set to `true` if you want to use insensitive-case searches when using `contains` or `starts_with` filters. Defaults to false.
+
 ## Contributing
 
 Make sure all the tests pass before sending a PR. To run the test suite, run `yarn test`. Please note that the codebase is using `dotenv` package to connect to a test db, so, to connect to your own, add a `.env` file inside the `tests` folder with the following structure:

package.json

@@ -1,6 +1,6 @@
 {
   "name": "knex-flex-filter",
-  "version": "0.3.0",
+  "version": "0.4.0-alpha-1",
   "description": "Flexible filtering and search for Knex queries",
   "main": "dist/index.js",
   "repository": "https://github.com/Terminal-Systems/knex-flex-filter",

src/index.js

@@ -12,6 +12,7 @@ export const STARTS_WITH = 'starts_with';
 export const NOT_STARTS_WITH = 'not_starts_with';
 export const ENDS_WITH = 'ends_with';
 export const NOT_ENDS_WITH = 'not_ends_with';
+export const SIMILAR_TO = 'similar_to';
 
 export const filterArray = [
   EQ,
@@ -25,6 +26,7 @@ export const filterArray = [
   STARTS_WITH,
   NOT_ENDS_WITH,
   ENDS_WITH,
+  SIMILAR_TO,
   NOT,
   GTE,
   LTE,
@@ -36,12 +38,13 @@ const conditionMap = {
   [LT]: '< ?',
   [NOT_IN]: '<> ANY(?)',
   [IN]: '= ANY(?)',
-  [NOT_CONTAINS]: "to_tsvector(??) @@ to_tsquery('!?')",
-  [CONTAINS]: "to_tsvector(??) @@ to_tsquery('?')",
+  [NOT_CONTAINS]: "NOT LIKE '%?%'",
+  [CONTAINS]: "LIKE '%?%'",
   [NOT_STARTS_WITH]: "NOT LIKE '?%'",
   [STARTS_WITH]: "LIKE '?%'",
   [NOT_ENDS_WITH]: "NOT LIKE '%?'",
   [ENDS_WITH]: "LIKE '%?'",
+  [SIMILAR_TO]: '% ?',
   [NOT]: '<> ?',
   [GTE]: '>= ?',
   [LTE]: '<= ?',
@@ -112,7 +115,9 @@ const processFilter = (filterQS, castFn, preprocessor) => {
 
 
 export const knexFlexFilter = (originalQuery, where = {}, opts = {}) => {
-  const { castFn, preprocessor = defaultPreprocessor(), isAggregateFn } = opts;
+  const {
+    castFn, preprocessor = defaultPreprocessor(), isAggregateFn, caseInsensitiveSearch = false,
+  } = opts;
 
   let result = originalQuery;
 
@@ -136,6 +141,10 @@ export const knexFlexFilter = (originalQuery, where = {}, opts = {}) => {
       query = query.replace(/(.*)'.*\?.*'(.*)/, '$1?$2');
     }
 
+    if (caseInsensitiveSearch) {
+      query = query.replace('LIKE', 'ILIKE');
+    }
+
     result = result[queryFn](query, [value]);
   });
 

tests/knex-flex-filter.test.js

@@ -114,23 +114,23 @@ describe('knex-flex-filter', () => {
     });
 
     it('correctly filters by contains', async (done) => {
-      const query = knexFlexFilter(knex.table('entities'), { name_contains: 'rick' }, { castFn });
+      const query = knexFlexFilter(knex.table('entities'), { name_contains: 'Ric' }, { castFn });
 
-      expect(query._statements[0].value.sql).toEqual('to_tsvector("name") @@ to_tsquery(?)');
-      expect(query._statements[0].value.bindings).toEqual(['rick']);
+      expect(query._statements[0].value.sql).toEqual('"name" LIKE ?');
+      expect(query._statements[0].value.bindings).toEqual(['%Ric%']);
 
       const result = await query;
 
       expect(result).toHaveLength(1);
-      expect(result[0].name).toContain('Rick');
+      expect(result[0].name).toContain('Ric');
       done();
     });
 
     it('correctly filters by not contains', async (done) => {
-      const query = knexFlexFilter(knex.table('entities'), { name_not_contains: 'rick' }, { castFn });
+      const query = knexFlexFilter(knex.table('entities'), { name_not_contains: 'Ric' }, { castFn });
 
-      expect(query._statements[0].value.sql).toEqual('to_tsvector("name") @@ to_tsquery(?)');
-      expect(query._statements[0].value.bindings).toEqual(['!rick']);
+      expect(query._statements[0].value.sql).toEqual('"name" NOT LIKE ?');
+      expect(query._statements[0].value.bindings).toEqual(['%Ric%']);
 
       const result = await query;
 
@@ -193,6 +193,36 @@ describe('knex-flex-filter', () => {
       done();
     });
 
+    it('correctly filters by similar_to', async (done) => {
+      const query = knexFlexFilter(
+        knex.table('entities'), { name_similar_to: 'jon doe' }, { castFn, caseInsensitiveSearch: true },
+      );
+
+      expect(query._statements[0].value.sql).toEqual('"name" % ?');
+      expect(query._statements[0].value.bindings).toEqual(['jon doe']);
+
+      const result = await query;
+
+      expect(result).toHaveLength(1);
+      expect(result[0].name).toEqual('John Doe');
+      done();
+    });
+
+    it('correctly filters by contains using case-insensitive search', async (done) => {
+      const query = knexFlexFilter(
+        knex.table('entities'), { name_contains: 'ric' }, { castFn, caseInsensitiveSearch: true },
+      );
+
+      expect(query._statements[0].value.sql).toEqual('"name" ILIKE ?');
+      expect(query._statements[0].value.bindings).toEqual(['%ric%']);
+
+      const result = await query;
+
+      expect(result).toHaveLength(1);
+      expect(result[0].name).toContain('Ric');
+      done();
+    });
+
     it('correctly filters by multiple filters at once', async (done) => {
       const query = knexFlexFilter(
         knex.table('entities'),
@@ -442,29 +472,29 @@ describe('knex-flex-filter', () => {
     it('correctly filters by contains', async (done) => {
       const query = knexFlexFilter(
         knex.table('entities'),
-        { name_contains: 'rick' },
+        { name_contains: 'Ric' },
         { castFn, preprocessor: jsonbPreprocessor('data') },
       );
 
-      expect(query._statements[0].value.sql).toEqual("to_tsvector(data->>'name') @@ to_tsquery(?)");
-      expect(query._statements[0].value.bindings).toEqual(['rick']);
+      expect(query._statements[0].value.sql).toEqual("data->>'name' LIKE ?");
+      expect(query._statements[0].value.bindings).toEqual(['%Ric%']);
 
       const result = await query;
 
       expect(result).toHaveLength(1);
-      expect(result[0].data.name).toContain('Rick');
+      expect(result[0].data.name).toContain('Ric');
       done();
     });
 
     it('correctly filters by not contains', async (done) => {
       const query = knexFlexFilter(
         knex.table('entities'),
-        { name_not_contains: 'rick' },
+        { name_not_contains: 'Ric' },
         { castFn, preprocessor: jsonbPreprocessor('data') },
       );
 
-      expect(query._statements[0].value.sql).toEqual("to_tsvector(data->>'name') @@ to_tsquery(?)");
-      expect(query._statements[0].value.bindings).toEqual(['!rick']);
+      expect(query._statements[0].value.sql).toEqual("data->>'name' NOT LIKE ?");
+      expect(query._statements[0].value.bindings).toEqual(['%Ric%']);
 
       const result = await query;
 
@@ -543,6 +573,42 @@ describe('knex-flex-filter', () => {
       done();
     });
 
+    it('correctly filters by similar_to', async (done) => {
+      const newCastFn = () => 'text';
+
+      const query = knexFlexFilter(
+        knex.table('entities'),
+        { name_similar_to: 'jon doe' },
+        { castFn: newCastFn, preprocessor: jsonbPreprocessor('data') },
+      );
+
+      expect(query._statements[0].value.sql).toEqual("(data->>'name')::text % ?");
+      expect(query._statements[0].value.bindings).toEqual(['jon doe']);
+
+      const result = await query;
+
+      expect(result).toHaveLength(1);
+      expect(result[0].data.name).toEqual('John Doe');
+      done();
+    });
+
+    it('correctly filters by contains using case-insensitive search', async (done) => {
+      const query = knexFlexFilter(
+        knex.table('entities'),
+        { name_contains: 'ric' },
+        { castFn, preprocessor: jsonbPreprocessor('data'), caseInsensitiveSearch: true },
+      );
+
+      expect(query._statements[0].value.sql).toEqual("data->>'name' ILIKE ?");
+      expect(query._statements[0].value.bindings).toEqual(['%ric%']);
+
+      const result = await query;
+
+      expect(result).toHaveLength(1);
+      expect(result[0].data.name).toContain('Ric');
+      done();
+    });
+
     it('correctly filters by multiple filters at once', async (done) => {
       const query = knexFlexFilter(
         knex.table('entities'),

tests/migrations/20190402152402_add_trgm_support.js

@@ -0,0 +1,9 @@
+exports.up = async (knex) => {
+  await knex.raw('CREATE EXTENSION pg_trgm');
+  await knex.raw('CREATE INDEX entities_name_trgm_index ON entities USING gin(name gin_trgm_ops)');
+};
+
+exports.down = async (knex) => {
+  await knex.raw('DROP EXTENSION pg_trgm');
+  await knex.raw('DROP INDEX entities_name_trgm_index');
+};