Move control of individual app scraping to UI (#12)

* Separate headset and search scrape * Update nodemon * Update UI to use new condensed routes * Update max search pages to 100 * Show current item in UI
KevinNovak · Nov 2, 2019 · 06dca9f · 06dca9f
1 parent f03950b
commit 06dca9f
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 29 deletions.
diff --git a/app.js b/app.js
@@ -16,6 +16,12 @@ function main() {
         res.json(gamePageData);
     });
 
+    _app.post('/api/headset-scrape', async (req, res) => {
+        let searchUrl = req.body.url;
+        let searchPageData = await _steamScraper.getHeadsetsFromAppPage(searchUrl);
+        res.json(searchPageData);
+    });
+
     _app.post('/api/search-scrape', async (req, res) => {
         let searchUrl = req.body.url;
         let searchPageData = await _steamScraper.getSearchPageData(searchUrl);

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -27,7 +27,7 @@
     "request-promise": "4.2.4"
   },
   "devDependencies": {
-    "nodemon": "^1.19.2"
+    "nodemon": "^1.19.4"
   },
   "homepage": "https://github.com/KevinNovak/vrgamedeals-Tools#readme",
   "repository": {

diff --git a/public/scripts/index.js b/public/scripts/index.js
@@ -4,7 +4,7 @@ const PRICE_NUMBER_REGEX = /\$(\d+\.\d{2})/;
 const PERCENT_NUMBER_REGEX = /(\d+)%/;
 
 const NEW_LINE = '&#10';
-const MAX_PAGES = 20;
+const MAX_PAGES = 100;
 
 const headsetAliases = {
     'Valve Index': {
@@ -105,21 +105,15 @@ async function retrieveSteamSearchTable() {
         let searchData = [];
         if (searchAllPages) {
             for (let i = 1; i <= MAX_PAGES; i++) {
-                let content = {
-                    url: `${steamSearchUrl}&page=${i}`
-                };
-                searchResultsDiv.innerHTML = `Retrieving page ${i}...`;
-                let searchPageData = await post('./api/search-scrape', content);
-                if (searchPageData.length < 1) {
-                    break;
+                let searchPageData = await retrieveSearchPageData(searchResultsDiv, steamSearchUrl, i);
+                if (searchPageData.length < 1) {	
+                    break;	
                 }
                 searchData.push(...searchPageData);
             }
         } else {
-            let content = {
-                url: steamSearchUrl
-            };
-            searchData = await post('./api/search-scrape', content);
+            let searchPageData = await retrieveSearchPageData(searchResultsDiv, steamSearchUrl);
+            searchData.push(...searchPageData);
         }
 
         if (!searchData || searchData.length < 1) {
@@ -145,6 +139,25 @@ async function retrieveSteamSearchTable() {
     retrieveSearchButton.disabled = false;
 }
 
+async function retrieveSearchPageData(searchResultsDiv, steamSearchUrl, pageNumber = 1) {
+    let content = {
+        url: `${steamSearchUrl}&page=${pageNumber}`
+    };
+    let searchPageData = await post('./api/search-scrape', content);
+    for (let [index, app] of searchPageData.entries()) {
+        let itemNumber = index + 1;
+        searchResultsDiv.innerHTML = `Retrieving page ${pageNumber}, result ${itemNumber} of ${searchPageData.length}...`;
+        app.headsets = [];
+        if (app.type == "APP") {
+            let content = {
+                url: app.link
+            };
+            app.headsets = await post('./api/headset-scrape', content);
+        }
+    }
+    return searchPageData;
+}
+
 function createMarkdownTable(searchData) {
     let header = '| Platform | Title | Price (USD) | Discount (%) | Rating (%) | Review Count |';
     let divider = '| :- | :- | -: | -: | -: | -: |';

diff --git a/services/steam-scraper.js b/services/steam-scraper.js
@@ -80,7 +80,7 @@ function getHeadsets($) {
     return headsets;
 }
 
-async function getHeadsetsFromLink(link) {
+async function getHeadsetsFromAppPage(link) {
     let pageHtml = await _rp({ url: link });
     let $ = _cheerio.load(pageHtml);
     return getHeadsets($);
@@ -98,7 +98,6 @@ async function getGameDataFromSearchResult(searchResult) {
     let percentOff = "";
     let reviewsPercent = "";
     let reviewsCount = "";
-    let headsets = [];
 
     title = $('div.search_name > span.title').text().trim();
     link = stripQueryString(searchResult.attribs.href);
@@ -125,8 +124,6 @@ async function getGameDataFromSearchResult(searchResult) {
             reviewsPercent = extractPercent(reviewsSummary);
             reviewsCount = extractReviewsCount(reviewsSummary).replace(/,/g, '');
         }
-
-        headsets = await getHeadsetsFromLink(link);
     }
 
     return {
@@ -139,7 +136,6 @@ async function getGameDataFromSearchResult(searchResult) {
         percentOff,
         reviewsPercent,
         reviewsCount,
-        headsets
     };
 }
 
@@ -179,5 +175,6 @@ function getGameDataFromGameElement(gameElement) {
 
 module.exports = {
     getAppPageData,
+    getHeadsetsFromAppPage,
     getSearchPageData
 };