issue 36: first pass at returning unparsed prefix

cmd/cli.js

@@ -12,7 +12,7 @@ console.log( input + '\n' );
 // -- search --
 console.time('search');
 var tokens = ph.tokenize( input );
-var results = ph.query( tokens );
+var results = ph.query( tokens ).ids;
 console.timeEnd('search');
 
 // print results

cmd/repl.js

@@ -11,7 +11,7 @@ var commands = {
   search: function( input, cb ){
     console.time('took');
     var tokens = ph.tokenize( input );
-    var results = ph.query( tokens );
+    var results = ph.query( tokens ).ids;
     ph.store.getMany( results, function( err, docs ){
       if( err ){ return console.error( err ); }
       docs.forEach( function( doc ){

prototype/query.js

@@ -5,26 +5,37 @@ var sorted = require('../lib/sorted');
 module.exports.queryOne = function( tokens ){
 
   var workingSet = [];
-  var validMatches = [];
-
-  var token = tokens.pop();
   var firstToken = true;
 
-  while( token ){
+  var res = {
+    ids: [],
+    match: {
+      token: '_NONE_MATCHED_',
+      position: 0
+    }
+  };
+
+  for( var i=tokens.length-1; i>=0; i-- ){
+
+    var token = tokens[i];
     var found = this.findToken( token );
 
     if( found.matches.length ){
       if( firstToken ){
         // console.log( 'initial' );
         workingSet = found.children;
-        validMatches = found.matches;
+        res.ids = found.matches;
+        res.match.token = token;
+        res.match.position = tokens.length - i;
       } else {
         // console.log( 'intersect' );
         var t = sorted.intersect([ found.matches, workingSet ]);
 
         if( t.length ){
-          validMatches = t;
+          res.ids = t;
           workingSet = sorted.intersect([ found.children, workingSet ]);
+          res.match.token = token;
+          res.match.position = tokens.length - i;
         }
         //else {
         // console.error( 'skip', token );
@@ -33,7 +44,7 @@ module.exports.queryOne = function( tokens ){
       }
     } else {
       // console.error( 'bailing out at', token );
-      return validMatches;
+      return res;
     }
 
     // console.log( 'found', {
@@ -43,17 +54,33 @@ module.exports.queryOne = function( tokens ){
     //   workingSet: workingSet.length
     // });
 
-    token = tokens.pop();
     firstToken = false;
   }
 
-  return validMatches;
+  return res;
 };
 
 module.exports.query = function( permutations ){
-  var matches = [];
+
+  var res = {
+    ids: [],
+    match: {
+      token: '_NONE_MATCHED_',
+      position: 0,
+      index: 0
+    }
+  };
+
   for( var p=0; p<permutations.length; p++ ){
-    matches = sorted.merge( matches, this.queryOne( permutations[p] ) );
+    var q = this.queryOne( permutations[p] );
+    res.ids = sorted.merge( res.ids, q.ids );
+
+    // select the most granular token from all permutations
+    if( q.match.position > res.match.position ){
+      res.match = q.match;
+      res.match.index = p;
+    }
   }
-  return matches;
+
+  return res;
 };

server/routes/query.js

@@ -5,7 +5,7 @@ module.exports = function( req, res ){
   var ph = req.app.locals.ph;
 
   var tokens = ph.tokenize( req.query.text );
-  var results = ph.query( tokens );
+  var results = ph.query( tokens ).ids;
 
   res.status(200).json( results );
 };

server/routes/search.js

@@ -1,12 +1,23 @@
 
+var analysis = require('../../lib/analysis');
+
 module.exports = function( req, res ){
 
   // placeholder
   var ph = req.app.locals.ph;
 
   // perform query
-  var tokens = ph.tokenize( req.query.text );
-  var ids = ph.query( tokens );
+  var permutations = ph.tokenize( req.query.text );
+  var q = ph.query( permutations );
+
+  // @todo: don't run tokenize again, it was previously run by ph.tokenize()
+  var tokens = analysis.tokenize( req.query.text );
+  var tokensAsString = tokens[q.match.index].join(' ');
+  var unparsedPrefix = tokensAsString.substring( 0, tokensAsString.indexOf( q.match.token )-1 );
+
+  console.error('tokens:', tokens);
+  console.error('match:', q.match);
+  console.error('unparsedPrefix:', unparsedPrefix);
 
   // language property
   var lang;
@@ -15,7 +26,7 @@ module.exports = function( req, res ){
   }
 
   // fetch all result docs by id
-  ph.store.getMany( ids, function( err, results ){
+  ph.store.getMany( q.ids, function( err, results ){
     if( err ){ return res.status(500).send(err); }
     if( !results || !results.length ){ return res.status(200).send([]); }
 

test/case.js

@@ -49,7 +49,7 @@ function runner( ph, actual, expected, next ){
   try {
     process.stderr.write('.');
     // console.time('took');
-    resultIds = ph.query( ph.tokenize( actual ) );
+    resultIds = ph.query( ph.tokenize( actual ) ).ids;
     // console.timeEnd('took');
     assert.ok( -1 !== resultIds.indexOf( expected ), 'id found in results' );
   } catch( e ){

test/functional.js

@@ -30,7 +30,7 @@ module.exports.tokenize = function(test, util) {
 // convenience function for writing quick 'n easy test cases
 function runner( test, ph, actual, expected ){
   test( actual, function(t) {
-    t.deepEqual( ph.query( ph.tokenize( actual ) ), expected );
+    t.deepEqual( ph.query( ph.tokenize( actual ) ).ids, expected );
     t.end();
   });
 }

test/prototype/query.js

@@ -18,7 +18,7 @@ module.exports.query = function(test, util) {
 // convenience function for writing quick 'n easy test cases
 function runner( test, ph, actual, expected ){
   test( actual, function(t) {
-    t.deepEqual( ph.query( actual ), expected );
+    t.deepEqual( ph.query( actual ).ids, expected );
     t.end();
   });
 }