Better error handling (#15)

* Better error handling * Refactoring to more services * Custom exceptions * URL input must start with steam URL
KevinNovak · Nov 27, 2019 · b47cd7d · b47cd7d
1 parent 6053c8e
commit b47cd7d
Show file tree

Hide file tree

Showing 6 changed files with 148 additions and 58 deletions.
diff --git a/app.js b/app.js
@@ -1,7 +1,9 @@
 const _express = require('express');
 const _app = _express();
 const _bodyParser = require('body-parser');
+const _httpClient = require('./services/http-client');
 const _steamScraper = require('./services/steam-scraper');
+const _logger = require('./services/logger');
 
 const PORT = process.env.PORT || 8080;
 
@@ -12,20 +14,79 @@ function main() {
 
     _app.post('/api/app-scrape', async (req, res) => {
         let appUrl = req.body.url;
-        let appPageData = await _steamScraper.getAppPageData(appUrl);
-        res.json(appPageData);
+
+        let appPageHtml;
+        try {
+            appPageHtml = await _httpClient.get(appUrl);
+        } catch (error) {
+            _logger.error(error);
+            res.status(500).json({ message: "Error retrieving page HTML." })
+            return;
+        }
+
+        try {
+            let appPageData = _steamScraper.getAppPageData(appPageHtml);
+            res.status(200).json(appPageData);
+            return;
+        } catch (error) {
+            if (error.type == "NO_GAME_ELEMENTS") {
+                res.status(400).json({ message: error.message });
+                return;
+            }
+
+            _logger.error(error);
+            res.status(500).json({ message: "Error scraping page data." });
+            return;
+        }
     });
 
     _app.post('/api/search-scrape', async (req, res) => {
         let searchUrl = req.body.url;
-        let searchPageData = await _steamScraper.getSearchPageData(searchUrl);
-        res.json(searchPageData);
+        let searchPageHtml;
+        try {
+            searchPageHtml = await _httpClient.get(searchUrl);
+        } catch (error) {
+            _logger.error(error);
+            res.status(500).json({ message: "Error retrieving page HTML." })
+            return;
+        }
+
+        try {
+            let searchPageData = _steamScraper.getSearchPageData(searchPageHtml);
+            res.status(200).json(searchPageData);
+            return;
+        } catch (error) {
+            _logger.error(error);
+            res.status(500).json({ message: "Error scraping page data." });
+            return;
+        }
     });
 
     _app.post('/api/search-app-scrape', async (req, res) => {
         let appUrl = req.body.url;
-        let appPageData = await _steamScraper.getSearchAppPageData(appUrl);
-        res.json(appPageData);
+        let appPageHtml;
+        try {
+            appPageHtml = await _httpClient.get(appUrl);
+        } catch (error) {
+            _logger.error(error);
+            res.status(500).json({ message: "Error retrieving page HTML." })
+            return;
+        }
+
+        try {
+            let appPageData = _steamScraper.getSearchAppPageData(appPageHtml);
+            res.status(200).json(appPageData);
+            return;
+        } catch (error) {
+            if (error.type == "NO_GAME_ELEMENTS") {
+                res.status(400).json({ message: error.message });
+                return;
+            }
+
+            _logger.error(error);
+            res.status(500).json({ message: "Error scraping page data." });
+            return;
+        }
     });
 
     _app.listen(PORT, () => {

diff --git a/models/exceptions.js b/models/exceptions.js
@@ -0,0 +1,8 @@
+function CustomException(type, message) {
+    this.type = type;
+    this.message = message;
+}
+
+module.exports = {
+    CustomException
+}
diff --git a/public/scripts/index.js b/public/scripts/index.js
@@ -1,10 +1,11 @@
-const STEAM_APP_URL_REGEX = /https:\/\/store.steampowered.com\/app\/\d+/;
-const STEAM_SEARCH_URL_REGEX = /https:\/\/store.steampowered.com\/search\/\S*/;
+const STEAM_APP_URL_REGEX = /^https:\/\/store.steampowered.com\/app\/\d+/;
+const STEAM_SEARCH_URL_REGEX = /^https:\/\/store.steampowered.com\/search\/\S*/;
 const PRICE_NUMBER_REGEX = /\$(\d+\.\d{2})/;
 const PERCENT_NUMBER_REGEX = /(\d+)%/;
 
 const NEW_LINE = '&#10';
 const MAX_PAGES = 100;
+const MAX_RETRIES = 10;
 
 const BUNDLE_PREFIX = "**Bundle** - ";
 
@@ -68,7 +69,7 @@ async function retrieveSteamAppTitle() {
 
         let link = document.createElement('a');
         link.innerText = text;
-        link.href = appData.link;
+        link.href = steamAppUrl;
         link.target = '_blank';
         link.style.display = 'inline';
 
@@ -247,19 +248,30 @@ function createMarkdownTable(searchData) {
 }
 
 async function post(url, content) {
-    let response = await fetch(
-        url,
-        {
-            method: 'POST',
-            headers: {
-                'Accept': 'application/json',
-                'Content-Type': 'application/json'
-            },
-            body: JSON.stringify(content)
-        });
+    let response;
+    for (i = 0; i < MAX_RETRIES; i++) {
+        response = await fetch(
+            url,
+            {
+                method: 'POST',
+                headers: {
+                    'Accept': 'application/json',
+                    'Content-Type': 'application/json'
+                },
+                body: JSON.stringify(content)
+            });
+        if (!shouldRetry(response.status)) {
+            break;
+        }
+    }
+
     return await response.json();
 }
 
+function shouldRetry(statusCode) {
+    return !((statusCode >= 200 && statusCode <= 299) || (statusCode >= 400 && statusCode <= 499));
+}
+
 function escapePipes(input) {
     return input.replace(/\|/g, '‖');
 }

diff --git a/services/http-client.js b/services/http-client.js
@@ -0,0 +1,9 @@
+const _rp = require('request-promise');
+
+async function get(url) {
+    return await _rp({ url });
+}
+
+module.exports = {
+    get
+};
diff --git a/services/logger.js b/services/logger.js
@@ -0,0 +1,7 @@
+function error(msg) {
+    console.log(msg)
+}
+
+module.exports = {
+    error
+}
diff --git a/services/steam-scraper.js b/services/steam-scraper.js
@@ -1,7 +1,7 @@
 const _cheerio = require('cheerio');
-const _rp = require('request-promise');
 const _regexUtils = require('../utils/regex-utils');
 const _stringUtils = require('../utils/string-utils');
+const { CustomException } = require('../models/exceptions');
 
 const TITLE_REMOVE = [
     'Buy',
@@ -11,75 +11,68 @@ const TITLE_REMOVE = [
     'Pre-Purchase'
 ];
 
-async function getSearchPageData(searchUrl) {
-    let searchPageHtml = await _rp({ url: searchUrl });
+function getAppPageData(appPageHtml) {
+    let firstGame = getMainGameElement(appPageHtml);
+    if (!firstGame) {
+        throw new CustomException("NO_GAME_ELEMENTS", "Could not find any game elements.");
+    }
+
+    let gameData = getGameDataFromGameElement(firstGame);
+    let countdown = getCountdownFromGameElement(firstGame);
+    let headsets = getHeadsets(appPageHtml);
+
+    return {
+        ...gameData,
+        countdown,
+        headsets
+    };
+}
+
+function getSearchPageData(searchPageHtml) {
     let $ = _cheerio.load(searchPageHtml);
 
     let searchResults = Array.from($('#search_resultsRows > a.search_result_row'));
 
     let searchPageData = [];
+
     for (let searchResult of searchResults) {
-        let gameData = await getGameDataFromSearchResult(searchResult);
+        let gameData = getGameDataFromSearchResult(searchResult);
         searchPageData.push(gameData);
     }
+
     return searchPageData;
 }
 
 
-async function getSearchAppPageData(appUrl) {
-    let appPageHtml = await _rp({ url: appUrl });
-    let $ = _cheerio.load(appPageHtml);
-
-    let firstGame = getMainGameElement($);
+function getSearchAppPageData(appPageHtml) {
+    let firstGame = getMainGameElement(appPageHtml);
     if (!firstGame) {
-        return {
-            error: true,
-            message: "Could not find any game elements."
-        };
+        throw new CustomException("NO_GAME_ELEMENTS", "Could not find any game elements.");
     }
 
     let countdown = getCountdownFromGameElement(firstGame);
-    let headsets = getHeadsets($);
+    let headsets = getHeadsets(appPageHtml);
 
     return {
         countdown,
         headsets
     };
 }
 
-async function getAppPageData(appUrl) {
-    let appPageHtml = await _rp({ url: appUrl });
+function getMainGameElement(appPageHtml) {
     let $ = _cheerio.load(appPageHtml);
 
-    let firstGame = getMainGameElement($);
-    if (!firstGame) {
-        return {
-            error: true,
-            message: "Could not find any game elements."
-        };
-    }
-
-    let gameData = getGameDataFromGameElement(firstGame);
-    let countdown = getCountdownFromGameElement(firstGame);
-    let headsets = getHeadsets($);
-
-    return {
-        link: appUrl,
-        ...gameData,
-        countdown,
-        headsets
-    };
-}
-
-function getMainGameElement($) {
     let gameElements = Array.from($('#game_area_purchase .game_area_purchase_game:not(.demo_above_purchase)'));
     if (gameElements.length < 1) {
         return;
     }
+
     return gameElements[0];
 }
 
-function getHeadsets($) {
+function getHeadsets(appPageHtml) {
+    let $ = _cheerio.load(appPageHtml);
+
     let headsetTitleElement = $('.details_block.vrsupport > div:contains("Headsets")').parent();
     let headsetElements = Array.from(headsetTitleElement.nextUntil('.details_block'));
 
@@ -95,7 +88,7 @@ function getHeadsets($) {
     return headsets;
 }
 
-async function getGameDataFromSearchResult(searchResult) {
+function getGameDataFromSearchResult(searchResult) {
     let $ = _cheerio.load(searchResult);
 
     let gameData = {