Improve user query terms hits (#1630)

* Some new deps
* This removes the HTML and MD from the `about` and stores in the limited `_about`.
* Still currently testing 1KiB term max *(combined)*
* Change default typing on `_description` to empty string.

NOTES:
* This could also be achieved with *marked* and *sanitize-html* however giving this a whirl for source isolation.

Post #1629 and applies to #1548

Auto-merge

README.md

@@ -70,6 +70,8 @@ Repository | Reference | Recent Version
 [passport-yahoo][passport-yahooGHUrl] <br />&#x22D4; [`OpenID2`][passport-yahooGHOpenIDUrl] | [Documentation][passport-yahooDOCUrl] | [![NPM version][passport-yahooNPMVersionImage]][passport-yahooNPMUrl] ![OpenID][openidLogo] [&#x22D4;][passport-openid]
 [pegjs][pegjsGHUrl] | [Documentation][pegjsDOCUrl] | [![NPM version][pegjsNPMVersionImage]][pegjsNPMUrl]
 [rate-limit-mongo][rate-limit-mongoGHUrl] | [Documentation][rate-limit-mongoDOCUrl] | [![NPM version][rate-limit-mongoNPMVersionImage]][rate-limit-mongoNPMUrl]
+[remark][remarkGHUrl] | [Documentation][remarkDOCUrl] | [![NPM version][remarkNPMVersionImage]][remarkNPMUrl]
+[remark-strip-html][remark-strip-htmlGHUrl] | [Documentation][remark-strip-htmlDOCUrl] | [![NPM version][remark-strip-htmlNPMVersionImage]][remark-strip-htmlNPMUrl]
 [request][requestGHUrl] | [Documentation][requestDOCUrl] | [![NPM version][requestNPMVersionImage]][requestNPMUrl]
 [rfc2047][rfc2047GHUrl] | [Documentation][rfc2047DOCUrl] | [![NPM version][rfc2047NPMVersionImage]][rfc2047NPMUrl]
 [S3rver][s3rverGHUrl] | [Documentation][s3rverDOCUrl] | [![NPM version][s3rverNPMVersionImage]][s3rverNPMUrl]
@@ -78,6 +80,7 @@ Repository | Reference | Recent Version
 [select2-bootstrap-css][select2-bootstrap-cssGHUrl] | [Documentation][select2-bootstrap-cssDOCUrl] | [![NPM version][select2-bootstrap-cssNPMVersionImage]][select2-bootstrap-cssNPMUrl]
 [serve-favicon][serve-faviconGHUrl] | [Documentation][serve-faviconDOCUrl] | [![NPM version][serve-faviconNPMVersionImage]][serve-faviconNPMUrl]
 [spdx-license-ids][spdx-license-idsGHUrl] | [Documentation][spdx-license-idsDOCUrl] | [![NPM version][spdx-license-idsNPMVersionImage]][spdx-license-idsNPMUrl]
+[strip-markdown][strip-markdownGHUrl] | [Documentation][strip-markdownDOCUrl] | [![NPM version][strip-markdownNPMVersionImage]][strip-markdownNPMUrl]
 [terser][terserGHUrl] | [Documentation][terserDOCUrl] | [![NPM version][terserNPMVersionImage]][terserNPMUrl]
 [toobusy-js][toobusy-jsGHUrl] <br />&#x22D4; [`harmony`][toobusy-jsGHUrlHarmonyUrl] | [Documentation][toobusy-jsDOCUrl] | [![NPM version][toobusy-jsNPMVersionImage]][toobusy-jsNPMUrl]
 [underscore][underscoreGHUrl] | [Documentation][underscoreDOCUrl] | [![NPM version][underscoreNPMVersionImage]][underscoreNPMUrl]
@@ -389,6 +392,16 @@ Outdated dependencies list can also be achieved with `$ npm --depth 0 outdated`
 [rate-limit-mongoNPMUrl]: https://www.npmjs.com/package/rate-limit-mongo
 [rate-limit-mongoNPMVersionImage]: https://img.shields.io/npm/v/rate-limit-mongo.svg?style=flat
 
+[remarkGHUrl]: https://github.com/remarkjs/remark
+[remarkDOCUrl]: https://github.com/remarkjs/remark/blob/master/readme.md
+[remarkNPMUrl]: https://www.npmjs.com/package/remark
+[remarkNPMVersionImage]: https://img.shields.io/npm/v/remark.svg?style=flat
+
+[remark-strip-htmlGHUrl]: https://github.com/craftzdog/remark-strip-html
+[remark-strip-htmlDOCUrl]: https://github.com/craftzdog/remark-strip-html/blob/master/readme.md
+[remark-strip-htmlNPMUrl]: https://www.npmjs.com/package/remark-strip-html
+[remark-strip-htmlNPMVersionImage]: https://img.shields.io/npm/v/remark-strip-html.svg?style=flat
+
 [requestGHUrl]: https://github.com/request/request
 [requestDOCUrl]: https://github.com/request/request/blob/master/README.md
 [requestNPMUrl]: https://www.npmjs.com/package/request
@@ -430,6 +443,11 @@ Outdated dependencies list can also be achieved with `$ npm --depth 0 outdated`
 [spdx-license-idsNPMUrl]: https://www.npmjs.com/package/spdx-license-ids
 [spdx-license-idsNPMVersionImage]: https://img.shields.io/npm/v/spdx-license-ids.svg?style=flat
 
+[strip-markdownGHUrl]: https://github.com/remarkjs/strip-markdown
+[strip-markdownDOCUrl]: https://github.com/remarkjs/strip-markdown/blob/master/readme.md
+[strip-markdownNPMUrl]: https://www.npmjs.com/package/strip-markdown
+[strip-markdownNPMVersionImage]: https://img.shields.io/npm/v/strip-markdown.svg?style=flat
+
 [terserGHUrl]: https://github.com/fabiosantoscode/terser
 [terserDOCUrl]: https://github.com/terser-js/terser/blob/master/README.md
 [terserNPMUrl]: https://www.npmjs.com/package/terser

controllers/script.js

@@ -11,6 +11,9 @@ var isDbg = require('../libs/debug').isDbg;
 var async = require('async');
 var _ = require('underscore');
 var SPDX = require('spdx-license-ids');
+var remark = require('remark');
+var stripHTML = require('remark-strip-html');
+var stripMD = require('strip-markdown');
 
 //--- Model inclusions
 var Discussion = require('../models/discussion').Discussion;
@@ -469,12 +472,25 @@ exports.edit = function (aReq, aRes, aNext) {
       } else if (typeof aReq.body.about !== 'undefined') {
         // POST
         aScript.about = aReq.body.about;
-        aScript._about = (aScript.about ? aScript.about.substr(0, 512) : null);
-        scriptGroups = (aReq.body.groups || '');
-        scriptGroups = scriptGroups.split(/,/);
-        addScriptToGroups(aScript, scriptGroups, function () {
-          aRes.redirect(script.scriptPageUri);
+
+        remark().use(stripHTML).use(stripMD).process(aScript.about, function(aErr, aFile) {
+          if (aErr || !aFile) {
+            aScript._about = (aScript.about ? aScript.about.substr(0, 512) : '');
+          } else {
+            aScript._about = (
+              aFile.contents
+                ? aFile.contents.replace(/(\r\n|\n|\r)+/gm, ' ').substr(0, 512)
+                : ''
+            );
+          }
+
+          scriptGroups = (aReq.body.groups || '');
+          scriptGroups = scriptGroups.split(/,/);
+          addScriptToGroups(aScript, scriptGroups, function () {
+            aRes.redirect(script.scriptPageUri);
+          });
         });
+
       } else {
         // GET
 

controllers/scriptStorage.js

@@ -1796,7 +1796,7 @@ exports.storeScript = function (aUser, aMeta, aBuf, aUpdate, aCallback) {
           // New script
           aScript = new Script({
             name: thisName,
-            _description: (thisDescription ? thisDescription.substr(0, 512) : null),
+            _description: (thisDescription ? thisDescription.substr(0, 512) : ''),
             author: aUser.name,
             installs: 0,
             rating: 0,
@@ -1833,7 +1833,7 @@ exports.storeScript = function (aUser, aMeta, aBuf, aUpdate, aCallback) {
             }), null);
             return;
           }
-          aScript._description = (thisDescription ? thisDescription.substr(0, 512) : null);
+          aScript._description = (thisDescription ? thisDescription.substr(0, 512) : '');
           aScript.meta = aMeta;
           aScript.uses = libraries;
 

package.json

@@ -54,6 +54,8 @@
     "passport-yahoo": "git://github.com/OpenUserJs/passport-yahoo.git#OpenID2",
     "pegjs": "0.10.0",
     "rate-limit-mongo": "1.0.3",
+    "remark": "10.0.1",
+    "remark-strip-html": "1.0.1",
     "request": "2.88.0",
     "rfc2047": "2.0.1",
     "s3rver": "3.2.0",
@@ -62,6 +64,7 @@
     "select2-bootstrap-css": "1.4.6",
     "serve-favicon": "2.5.0",
     "spdx-license-ids": "3.0.4",
+    "strip-markdown": "3.0.4",
     "terser": "4.0.0",
     "toobusy-js": "0.5.1",
     "underscore": "1.9.1",