From 067d205e853208deccf55816fb091cbb13a1ba2d Mon Sep 17 00:00:00 2001 From: ushmajit Date: Fri, 11 Mar 2022 15:07:43 +0530 Subject: [PATCH] Implementation of search and getPlugins API --- README.md | 89 ++++++++++++++----- package.json | 24 +++-- src/app.js | 148 ++++++++++++++++++++++++++++++ src/elastic_search_helper.js | 168 +++++++++++++++++++++++++++++++++++ src/env.js | 3 + src/index.js | 23 ----- src/npm_helper.js | 86 ++++++++++++++++++ src/request_validator.js | 79 ++++++++++++++++ 8 files changed, 565 insertions(+), 55 deletions(-) create mode 100644 src/app.js create mode 100644 src/elastic_search_helper.js create mode 100644 src/env.js delete mode 100644 src/index.js create mode 100644 src/npm_helper.js create mode 100644 src/request_validator.js diff --git a/README.md b/README.md index 743d7c9..5543c69 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,73 @@ -# template-nodejs -A template project for nodejs. Has integrated linting, testing, -coverage, reporting, GitHub actions for publishing to npm repository, dependency updates and other goodies. - -Easily use this template to quick start a production ready nodejs project template. +# plugin-registry-backend +Plugin Registry Backend service is a express JS service which supports clients +like Brackets Code Editor (https://brackets.io/) and Phoenix Code Editor(https://phoenix.core.ai/) +by providing all the necessary APIs to render the in-app extension store. ## Code Guardian -[![ build verification](https://github.com/aicore/template-nodejs/actions/workflows/build_verify.yml/badge.svg)](https://github.com/aicore/template-nodejs/actions/workflows/build_verify.yml) - - - Sonar code quality check - Security rating - vulnerabilities - Code Coverage - Code Bugs - Reliability Rating - Maintainability Rating - Lines of Code - Technical debt +[![ build verification](https://github.com/aicore/plugin-registry-backend/actions/workflows/build_verify.yml/badge.svg)](https://github.com/aicore/template-nodejs/actions/workflows/build_verify.yml) + + + Sonar code quality check + Security rating + vulnerabilities + Code Coverage + Code Bugs + Reliability Rating + Maintainability Rating + Lines of Code + Technical debt -# TODOs after template use -1. Update package.json with your app defaults -2. Check Build actions on pull requests. -3. In sonar cloud, enable Automatic analysis from `Administration - Analysis Method` for the first time before a pull request is raised: ![image](https://user-images.githubusercontent.com/5336369/148695840-65585d04-5e59-450b-8794-54ca3c62b9fe.png) -4. Check codacy runs on pull requests, set codacy defaults. You may remove codacy if sonar cloud is only needed. -5. Update the above Code Guardian badges; change all `id=aicore_template-nodejs-ts` to the sonar id of your project fields. +# API Documentation + +## SEARCH API + +This API supports text-based search queries (see below for detailed params) on plugin registry. Search API can retrieve plugins(Extensions/Themes) by searching titles, keywords or authornames(by default). We can also search on more fields apart from the previously mentioned by passing it as a param. This API can be a functional entry point for all the UI search-box usecases. + +### Parameters + +* **Required Parameters** + + * **clientID {String}**: A unique identifier required for authenticating clients + * **query {String}**: String containing the text to be searched +* **Optional Parameters** + + * **filters {Object} (Default: Empty)**: Object contaning the fields array and sortBy field. See below e.g + + * **fields {String Array}**: Array containing additional fields to be searched. + * **sortyBy {String}**: Constant value(asc/desc) to sort the results by total number of downloads. + E.g, + + ``` const filter = { fields:['metadata.author.name','metadata.author.email'], sortBy:'desc'}``` + * **resultSize {Integer} (Default: 10)** : SearchResults list size. + * **skipIndex {Integer} {Default: 0}**: Integer value required for pagination + +### Sample Request + +``` curl -d '{"clientID":"your_iD","query":"Python"}' -H "Content-Type: application/json" http://localhost:3000/search ``` + +## getPlugins API + +This API can be used to retrieve plugins by assetType(Extension/Theme) (see below for detailed params) on plugin registry. We can also apply certain keywords as filters to refine our search results. This API can be a functional entry point for loading plugins by default in the UI.r all the UI search-box usecases. + +### Parameters + +* **Required Parameters** + + * **clientID {String}**: A unique identifier required for authenticating clients + * **assetType {String}**: accepted values are 'EXTENSION' or 'THEME' +* **Optional Parameters** + + * **filters {Object} (Default: Empty)**: Object contaning the fields array and sortBy field. See below e.g + + * **keywords {String Array}**: Array containing additional keywords to match. + * **sortyBy {String}**: Constant value(asc/desc) to sort the results by total number of downloads. + E.g, + + ``` const filter = { keywords:['HTML','HTML5'], sortBy:'desc'}``` + * **resultSize {Integer} (Default: 10)** : SearchResults list size. + * **skipIndex {Integer} {Default: 0}**: Integer value required for pagination # Commands available diff --git a/package.json b/package.json index b578cd2..89bf001 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,13 @@ { - "name": "@aicore/template-nodejs", + "name": "@aicore/plugin-registry-backend", "version": "1.0.4", "description": "Template for nodejs with unit gulp build, test, coverage, code guardian, github and Other defaults", - "main": "index.js", + "main": "src/app.js", "type": "module", "keywords": [ - "template", + "backend", + "RestAPI", + "API", "nodejs", "unit", "testing", @@ -36,14 +38,14 @@ ], "repository": { "type": "git", - "url": "git+https://github.com/aicore/template-nodejs.git" + "url": "git+https://github.com/aicore/plugin-registry-backend.git" }, - "author": "Arun, core.ai", + "author": "Ushmajit, core.ai", "license": "AGPL-3.0-or-later", "bugs": { - "url": "https://github.com/aicore/template-nodejs/issues" + "url": "https://github.com/aicore/plugin-registry-backend/issues" }, - "homepage": "https://github.com/aicore/template-nodejs#readme", + "homepage": "https://github.com/aicore/plugin-registry-backend#readme", "devDependencies": { "@commitlint/cli": "16.2.1", "@commitlint/config-conventional": "16.2.1", @@ -53,5 +55,11 @@ "husky": "7.0.4", "mocha": "9.2.1" }, - "dependencies": {} + "dependencies": { + "@aicore/elasticsearch-lib": "^1.0.1", + "cors": "^2.8.5", + "dotenv": "^16.0.0", + "express": "^4.17.3", + "superagent": "^7.1.1" + } } diff --git a/src/app.js b/src/app.js new file mode 100644 index 0000000..5e4347c --- /dev/null +++ b/src/app.js @@ -0,0 +1,148 @@ +/* + * GNU AGPL-3.0 License + * + * Copyright (c) 2021 - present core.ai . All rights reserved. + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Affero General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along + * with this program. If not, see https://opensource.org/licenses/AGPL-3.0. + * + */ +import './env.js'; +import express from 'express'; +import cors from 'cors'; +import requestValidator from './request_validator.js'; +import searchHelper from './elastic_search_helper.js'; +import npmHelper from './npm_helper.js'; + +const app = express(); +const port = 3000; +let filters = {}; +let resultSize = 10; +let skipIndex = 0; + +app.use(express.json()); + +app.use(cors({ + methods: ['GET', 'POST'] +})); + +/** + * SEARCH API : This API supports text-based search queries (see below for detailed params) + * on plugin registry. Search API can retrieve plugins(Extensions/Themes) by searching titles, + * keywords or authornames(by default). We can also search on more fields apart from the previously mentioned + * by passing it as a param. This API can be a functional entry point for all the UI search-box usecases. + * + * Parameters: + * Required: + * 1. clientID {String}: A unique identifier required for authenticating clients + * 2. query {String}: String containing the text to be searched + * + * Optional: + * 1. filters {Object} (Default: Empty) : Object contaning the fields array and sortBy field. See below e.g + * a. fieldsArray {String Array}: Array containing additional fields to be searched + * b. sortyBy {String}: Constant value(asc/desc) to sort the results by total + * number of downloads. + * E.g const filter = { fields:['metadata.author.name','metadata.author.email'], sortBy:'desc'} + * 2. resultSize {Integer} (Default: 10) : SearchResults list size + * 3. skipIndex {Integer} {Default: 0} : Integer value required for pagination + * + */ +app.post('/search', async function(req, res) { + let validationResponse = await requestValidator.validateSearchRequest(req); + + res.set({ + 'Access-Control-Allow-Origin': '*' + }); + + if (!validationResponse.isValid) { + res.status(validationResponse.statusCode); + res.json({error: validationResponse.errorMessage}) + return ; + } + + const searchQuery = req.body["query"]; + assignOptionalParameters(req); + + try { + let searchResponse = await searchHelper.performTextSearch(searchQuery, filters, resultSize, skipIndex); + let modifiedResponse = await npmHelper.updateDownloadStats(searchResponse); + console.log("Success: Search API Request succeeded with no errors, Please check logs for response"); + res.status(200); + res.json(modifiedResponse); + } catch (error) { + res.status(500); + res.json({error: JSON.stringify(error)}); + } +}); + +/** + * getPlugins API : This API can be used to retrieve plugins by assetType(Extension/Theme) (see below for detailed params) + * on plugin registry. We can also apply certain keywords as filters to refine our search results. + * This API can be a functional entry point for loading plugins by default in the UI. + * + * Parameters: + * Required: + * 1. clientID {String}: A unique identifier required for authenticating clients + * 2. assetType {String}: accepted values are 'EXTENSION' or 'THEME' + * + * Optional: + * 1. filters {Object} (Default: Empty) : Object contaning the fields array and sortBy field. See below e.g + * a. keywordsArray {String Array}: Array containing additional keywords to match + * b. sortyBy {String}: Constant value(asc/desc) to sort the results by total + * number of downloads. + * E.g const filter = { keywords:['HTML', 'HTML5'], sortBy:'desc'} + * 2. resultSize {Integer} (Default: 10) : SearchResults list size + * 3. skipIndex {Integer} {Default: 0} : Integer value required for pagination + * + */ +app.post('/getPlugins', async function(req, res) { + let validationResponse = await requestValidator.validateGetPluginsRequest(req); + + res.set({ + 'Access-Control-Allow-Origin': '*' + }); + + if (!validationResponse.isValid) { + res.status(validationResponse.statusCode); + res.json({error: validationResponse.errorMessage}) + return ; + } + + const assetType = req.body["assetType"]; + assignOptionalParameters(req); + + try { + let pluginsResponse = await searchHelper.getPlugins(assetType, filters, resultSize, skipIndex); + let modifiedResponse = await npmHelper.updateDownloadStats(pluginsResponse); + console.log("Success: GetPlugins API Request succeeded with no errors, Please check logs for response"); + res.status(200); + res.json(modifiedResponse); + } catch (error) { + res.status(500); + res.json({error: JSON.stringify(error)}); + } +}); + +app.listen(port, () => { + console.log(`Plugin-Registry-Backend server listening at http://localhost:${port}`); +}); + +async function assignOptionalParameters(req) { + if (req.body.hasOwnProperty('filters')) { + filters = req.body["filters"]; + } + if (req.body.hasOwnProperty("resultSize")) { + resultSize = req.body.resultSize; + } + if (req.body.hasOwnProperty("skipIndex")) { + resultSize = req.body.resultSize; + } +} diff --git a/src/elastic_search_helper.js b/src/elastic_search_helper.js new file mode 100644 index 0000000..43b2a38 --- /dev/null +++ b/src/elastic_search_helper.js @@ -0,0 +1,168 @@ +/* + * GNU AGPL-3.0 License + * + * Copyright (c) 2021 - present core.ai . All rights reserved. + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Affero General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along + * with this program. If not, see https://opensource.org/licenses/AGPL-3.0. + * + */ +import './env.js'; +import searchClient from '@aicore/elasticsearch-lib'; + +const INDEX_NAME = process.env.ELASTIC_SEARCH_INDEX_NAME; +const NODE_ADDRESS = process.env.ELASTIC_SEARCH_HOST_ADDRESS; +const PAGE_SIZE = parseInt(process.env.ELASTIC_SEARCH_PAGE_SIZE); +/** + * Accepts the necessary params to construct the search query object, + * applies the passed filters and returns the result object from elastic search. + * @param {String} query (Required) + * @param {Object} filters (Optional) + * @param {Integer} resultSize (Optional) + * @param {Integer} skipIndex (Optional) + * @returns {Object} + */ +async function performTextSearch(query, filters = {}, + resultSize = PAGE_SIZE, skipIndex = 0) { + + if (!query) { + throw new Error("Invalid Request: text query is a required parameter."); + } + let shouldArray = [ + { match: { 'metadata.title': query } }, + { match: { 'metadata.name': query } }, + { match: { 'metadata.keywords': query } }, + { match: { 'metadata.author.name': query } } + ]; + + if (filters.hasOwnProperty('fields')) { + filters.fields.forEach(field => { + shouldArray.push({ match: { [field] : query }}); + }); + } + + if (filters.hasOwnProperty('assetType')) { + shouldArray.push({ match: { 'metadata.assetType' : filters.assetType }}); + } + + let boolObj = { + should: shouldArray + }; + + let searchQuery = { + index: INDEX_NAME, + from: skipIndex, + size: resultSize, + body: { + query: { + nested: { + path: 'metadata', + query: { + bool: boolObj + } + } + } + } + }; + + if (filters.hasOwnProperty('sortBy')) { + searchQuery = { + index: INDEX_NAME, + size: resultSize, + body: { + query: { + nested: { + path: 'metadata', + query: { + bool: boolObj + } + } + }, + sort: [{ 'totalDownloads': { 'order': filters.sortBy} }] + } + }; + } + + let response = await searchClient.search(NODE_ADDRESS, INDEX_NAME, searchQuery); + return response; +} +/** + * Accepts the necessary params to construct the search query object to retrieve plugins, + * applies the passed filters and returns the result object from elastic search. + * @param {String} assetType (Required) + * @param {Object} filters (Optional) + * @param {Integer} resultSize (Optional) + * @param {Integer} skipIndex (Optional) + * @returns {Object} + */ +async function getPlugins(assetType, filters = {}, resultSize = PAGE_SIZE, skipIndex = 0) { + + if (!assetType) { + throw new Error("Invalid Request: assetType is a required parameter."); + } + + let mustArray = [ + { match: { 'metadata.assetType' : assetType } } + ]; + + if (filters.hasOwnProperty('keywords')) { + + filters.keywords.forEach(keyword => { + mustArray.push({ match_phrase: { 'metadata.keywords': keyword } }); + }); + } + + let searchQuery = { + index: INDEX_NAME, + from: skipIndex, + size: resultSize, + body: { + query: { + nested: { + path: 'metadata', + query: { + bool: { + must: mustArray + } + } + } + } + } + }; + + if (filters.hasOwnProperty('sortBy')) { + searchQuery = { + index: INDEX_NAME, + from: skipIndex, + size: resultSize, + body: { + query: { + nested: { + path: 'metadata', + query: { + bool: { + must: mustArray + } + } + } + }, + sort: [{ 'totalDownloads': { 'order': filters.sortBy} }] + } + }; + } + let response = await searchClient.search(NODE_ADDRESS, INDEX_NAME, searchQuery); + return response; +} + +export default { + performTextSearch, + getPlugins +}; \ No newline at end of file diff --git a/src/env.js b/src/env.js new file mode 100644 index 0000000..4341323 --- /dev/null +++ b/src/env.js @@ -0,0 +1,3 @@ +// Refer : https://github.com/motdotla/dotenv/issues/133#issuecomment-255298822 +import dotenv from 'dotenv'; +dotenv.config(); \ No newline at end of file diff --git a/src/index.js b/src/index.js deleted file mode 100644 index f829bc6..0000000 --- a/src/index.js +++ /dev/null @@ -1,23 +0,0 @@ -/* - * GNU AGPL-3.0 License - * - * Copyright (c) 2021 - present core.ai . All rights reserved. - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU Affero General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; - * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License along - * with this program. If not, see https://opensource.org/licenses/AGPL-3.0. - * - */ - -function helloWorld(name) { - return "Hello World " + name; -} - -export default helloWorld; diff --git a/src/npm_helper.js b/src/npm_helper.js new file mode 100644 index 0000000..3fea418 --- /dev/null +++ b/src/npm_helper.js @@ -0,0 +1,86 @@ +/* + * GNU AGPL-3.0 License + * + * Copyright (c) 2021 - present core.ai . All rights reserved. + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Affero General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along + * with this program. If not, see https://opensource.org/licenses/AGPL-3.0. + * + */ +import './env.js'; +import request from "superagent"; + +const NPM_PACKAGE_PREFIX = '@phoenix-plugin-registry/'; +const NPM_STATS_START_DATE = process.env.NPM_STATS_START_DATE; +const CURR_DATE = new Date().toISOString().slice(0, 10); +/** + * Updates the totalDownload field by retrieving download data from NPM and returns the modified list. + * @param {Object Array} searchList + * @returns {Object} + */ +async function updateDownloadStats(searchList) { + let modifiedList = [], downloadList = []; + if (!searchList) { + throw new Error("Error in NPM Helper: Undefined search results passed as param"); + } + let i = 0; + for (i = 0; i < searchList.length; i++) { + let element = searchList[i]; + if (element.hasOwnProperty('_source') && element._source.hasOwnProperty('totalDownloads') + && element._source.hasOwnProperty('metadata')) { + try { + let packageName = NPM_PACKAGE_PREFIX + searchList[i]._source.metadata.name; + let npmTotalDownloads = 0; + let npmResponse = await getNpmStats(packageName, NPM_STATS_START_DATE, CURR_DATE); + npmTotalDownloads = npmResponse.body.downloads; + element._source.npmDownloads = npmTotalDownloads; + element._source.totalDownloads += npmTotalDownloads; + } catch (err) { + console.error("Error Updating Download data for Package :" + packageName + + " Reason: " + JSON.stringify(err)); + } + } + modifiedList.push(element); + } + return modifiedList; +} +/** + * Returns the download stats from NPM for a particular package within a specified period. + * @param {String} pkg + * @param {String} start + * @param {String} end + * @returns {Object} + */ +async function getNpmStats(pkg, start, end) { + const url = `https://api.npmjs.org/downloads/point/${start}:${end}/${pkg ? pkg : ""}`; + try { + const { res, body } = await request + .get(url) + .timeout({ + response: 3 * 1000, + deadline: 5 * 1000, + }); + if (!res && res.hasOwnProperty('statusCode') && res.statusCode === 400) { + throw new Error("Error retrieving details from NPM API"); + } + return { + statusCode: res.statusCode, + body: body, + }; + } catch (err) { + throw err; + } +} + +export default { + updateDownloadStats, + getNpmStats +}; diff --git a/src/request_validator.js b/src/request_validator.js new file mode 100644 index 0000000..fd85426 --- /dev/null +++ b/src/request_validator.js @@ -0,0 +1,79 @@ +/* + * GNU AGPL-3.0 License + * + * Copyright (c) 2021 - present core.ai . All rights reserved. + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Affero General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along + * with this program. If not, see https://opensource.org/licenses/AGPL-3.0. + * + */ +import './env.js'; + +const REGISTERED_CLIENT_IDS = process.env.REGISTERED_CLIENT_IDS.split(", "); + +async function validateGetPluginsRequest(req) { + let clientIP= req.headers["x-real-ip"] || req.headers['X-Forwarded-For'] || req.socket.remoteAddress; + + console.log("Received getPluginsRequest from clientIP :" + clientIP); + + const validationResponse = { + isValid: true, + statusCode: 200, + errorMessage: "" + }; + + if (!req.body || !req.body.hasOwnProperty("assetType") || !req.body.hasOwnProperty("clientID")) { + validationResponse.isValid = false; + validationResponse.statusCode = 400; + validationResponse.errorMessage = "Invalid Argument: clientID & assetType are mandatory parameters."; + return validationResponse; + } + + if (!REGISTERED_CLIENT_IDS.includes(req.body["clientID"])) { + validationResponse.isValid = false; + validationResponse.statusCode = 403; + validationResponse.errorMessage = "AuthenticationError: Not Authorised to access getPlugins API."; + } + + return validationResponse; +} + +async function validateSearchRequest(req) { + let clientIP= req.headers["x-real-ip"] || req.headers['X-Forwarded-For'] || req.socket.remoteAddress; + + console.log("Received getPluginsRequest from clientIP : " + clientIP + " Request : " + JSON.stringify(req.body)); + + const validationResponse = { + isValid: true, + statusCode: 200, + errorMessage: "" + }; + + if (!req.body || !req.body.hasOwnProperty("query") || !req.body.hasOwnProperty("clientID")) { + validationResponse.isValid = false; + validationResponse.statusCode = 400; + validationResponse.errorMessage = "Invalid Argument: clientID & query are mandatory parameters."; + return validationResponse; + } + + if (!REGISTERED_CLIENT_IDS.includes(req.body["clientID"])) { + validationResponse.isValid = false; + validationResponse.statusCode = 403; + validationResponse.errorMessage = "AuthenticationError: Not Authorised to access getPlugins API."; + } + + return validationResponse; +} + +export default { + validateGetPluginsRequest, + validateSearchRequest +}; \ No newline at end of file