diff --git a/app.js b/app.js index e8bafd9..f685a95 100644 --- a/app.js +++ b/app.js @@ -1,7 +1,9 @@ const _express = require('express'); const _app = _express(); const _bodyParser = require('body-parser'); +const _httpClient = require('./services/http-client'); const _steamScraper = require('./services/steam-scraper'); +const _logger = require('./services/logger'); const PORT = process.env.PORT || 8080; @@ -12,20 +14,79 @@ function main() { _app.post('/api/app-scrape', async (req, res) => { let appUrl = req.body.url; - let appPageData = await _steamScraper.getAppPageData(appUrl); - res.json(appPageData); + + let appPageHtml; + try { + appPageHtml = await _httpClient.get(appUrl); + } catch (error) { + _logger.error(error); + res.status(500).json({ message: "Error retrieving page HTML." }) + return; + } + + try { + let appPageData = _steamScraper.getAppPageData(appPageHtml); + res.status(200).json(appPageData); + return; + } catch (error) { + if (error.type == "NO_GAME_ELEMENTS") { + res.status(400).json({ message: error.message }); + return; + } + + _logger.error(error); + res.status(500).json({ message: "Error scraping page data." }); + return; + } }); _app.post('/api/search-scrape', async (req, res) => { let searchUrl = req.body.url; - let searchPageData = await _steamScraper.getSearchPageData(searchUrl); - res.json(searchPageData); + let searchPageHtml; + try { + searchPageHtml = await _httpClient.get(searchUrl); + } catch (error) { + _logger.error(error); + res.status(500).json({ message: "Error retrieving page HTML." }) + return; + } + + try { + let searchPageData = _steamScraper.getSearchPageData(searchPageHtml); + res.status(200).json(searchPageData); + return; + } catch (error) { + _logger.error(error); + res.status(500).json({ message: "Error scraping page data." }); + return; + } }); _app.post('/api/search-app-scrape', async (req, res) => { let appUrl = req.body.url; - let appPageData = await _steamScraper.getSearchAppPageData(appUrl); - res.json(appPageData); + let appPageHtml; + try { + appPageHtml = await _httpClient.get(appUrl); + } catch (error) { + _logger.error(error); + res.status(500).json({ message: "Error retrieving page HTML." }) + return; + } + + try { + let appPageData = _steamScraper.getSearchAppPageData(appPageHtml); + res.status(200).json(appPageData); + return; + } catch (error) { + if (error.type == "NO_GAME_ELEMENTS") { + res.status(400).json({ message: error.message }); + return; + } + + _logger.error(error); + res.status(500).json({ message: "Error scraping page data." }); + return; + } }); _app.listen(PORT, () => { diff --git a/models/exceptions.js b/models/exceptions.js new file mode 100644 index 0000000..0ee580b --- /dev/null +++ b/models/exceptions.js @@ -0,0 +1,8 @@ +function CustomException(type, message) { + this.type = type; + this.message = message; +} + +module.exports = { + CustomException +} \ No newline at end of file diff --git a/public/scripts/index.js b/public/scripts/index.js index 05246cd..192ffcf 100644 --- a/public/scripts/index.js +++ b/public/scripts/index.js @@ -1,10 +1,11 @@ -const STEAM_APP_URL_REGEX = /https:\/\/store.steampowered.com\/app\/\d+/; -const STEAM_SEARCH_URL_REGEX = /https:\/\/store.steampowered.com\/search\/\S*/; +const STEAM_APP_URL_REGEX = /^https:\/\/store.steampowered.com\/app\/\d+/; +const STEAM_SEARCH_URL_REGEX = /^https:\/\/store.steampowered.com\/search\/\S*/; const PRICE_NUMBER_REGEX = /\$(\d+\.\d{2})/; const PERCENT_NUMBER_REGEX = /(\d+)%/; const NEW_LINE = ' '; const MAX_PAGES = 100; +const MAX_RETRIES = 10; const BUNDLE_PREFIX = "**Bundle** - "; @@ -68,7 +69,7 @@ async function retrieveSteamAppTitle() { let link = document.createElement('a'); link.innerText = text; - link.href = appData.link; + link.href = steamAppUrl; link.target = '_blank'; link.style.display = 'inline'; @@ -247,19 +248,30 @@ function createMarkdownTable(searchData) { } async function post(url, content) { - let response = await fetch( - url, - { - method: 'POST', - headers: { - 'Accept': 'application/json', - 'Content-Type': 'application/json' - }, - body: JSON.stringify(content) - }); + let response; + for (i = 0; i < MAX_RETRIES; i++) { + response = await fetch( + url, + { + method: 'POST', + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json' + }, + body: JSON.stringify(content) + }); + if (!shouldRetry(response.status)) { + break; + } + } + return await response.json(); } +function shouldRetry(statusCode) { + return !((statusCode >= 200 && statusCode <= 299) || (statusCode >= 400 && statusCode <= 499)); +} + function escapePipes(input) { return input.replace(/\|/g, '‖'); } diff --git a/services/http-client.js b/services/http-client.js new file mode 100644 index 0000000..9bcd0d9 --- /dev/null +++ b/services/http-client.js @@ -0,0 +1,9 @@ +const _rp = require('request-promise'); + +async function get(url) { + return await _rp({ url }); +} + +module.exports = { + get +}; \ No newline at end of file diff --git a/services/logger.js b/services/logger.js new file mode 100644 index 0000000..f929fc1 --- /dev/null +++ b/services/logger.js @@ -0,0 +1,7 @@ +function error(msg) { + console.log(msg) +} + +module.exports = { + error +} \ No newline at end of file diff --git a/services/steam-scraper.js b/services/steam-scraper.js index 838bdfd..c9038bc 100644 --- a/services/steam-scraper.js +++ b/services/steam-scraper.js @@ -1,7 +1,7 @@ const _cheerio = require('cheerio'); -const _rp = require('request-promise'); const _regexUtils = require('../utils/regex-utils'); const _stringUtils = require('../utils/string-utils'); +const { CustomException } = require('../models/exceptions'); const TITLE_REMOVE = [ 'Buy', @@ -11,35 +11,47 @@ const TITLE_REMOVE = [ 'Pre-Purchase' ]; -async function getSearchPageData(searchUrl) { - let searchPageHtml = await _rp({ url: searchUrl }); +function getAppPageData(appPageHtml) { + let firstGame = getMainGameElement(appPageHtml); + if (!firstGame) { + throw new CustomException("NO_GAME_ELEMENTS", "Could not find any game elements."); + } + + let gameData = getGameDataFromGameElement(firstGame); + let countdown = getCountdownFromGameElement(firstGame); + let headsets = getHeadsets(appPageHtml); + + return { + ...gameData, + countdown, + headsets + }; +} + +function getSearchPageData(searchPageHtml) { let $ = _cheerio.load(searchPageHtml); let searchResults = Array.from($('#search_resultsRows > a.search_result_row')); let searchPageData = []; + for (let searchResult of searchResults) { - let gameData = await getGameDataFromSearchResult(searchResult); + let gameData = getGameDataFromSearchResult(searchResult); searchPageData.push(gameData); } + return searchPageData; } -async function getSearchAppPageData(appUrl) { - let appPageHtml = await _rp({ url: appUrl }); - let $ = _cheerio.load(appPageHtml); - - let firstGame = getMainGameElement($); +function getSearchAppPageData(appPageHtml) { + let firstGame = getMainGameElement(appPageHtml); if (!firstGame) { - return { - error: true, - message: "Could not find any game elements." - }; + throw new CustomException("NO_GAME_ELEMENTS", "Could not find any game elements."); } let countdown = getCountdownFromGameElement(firstGame); - let headsets = getHeadsets($); + let headsets = getHeadsets(appPageHtml); return { countdown, @@ -47,39 +59,20 @@ async function getSearchAppPageData(appUrl) { }; } -async function getAppPageData(appUrl) { - let appPageHtml = await _rp({ url: appUrl }); +function getMainGameElement(appPageHtml) { let $ = _cheerio.load(appPageHtml); - let firstGame = getMainGameElement($); - if (!firstGame) { - return { - error: true, - message: "Could not find any game elements." - }; - } - - let gameData = getGameDataFromGameElement(firstGame); - let countdown = getCountdownFromGameElement(firstGame); - let headsets = getHeadsets($); - - return { - link: appUrl, - ...gameData, - countdown, - headsets - }; -} - -function getMainGameElement($) { let gameElements = Array.from($('#game_area_purchase .game_area_purchase_game:not(.demo_above_purchase)')); if (gameElements.length < 1) { return; } + return gameElements[0]; } -function getHeadsets($) { +function getHeadsets(appPageHtml) { + let $ = _cheerio.load(appPageHtml); + let headsetTitleElement = $('.details_block.vrsupport > div:contains("Headsets")').parent(); let headsetElements = Array.from(headsetTitleElement.nextUntil('.details_block')); @@ -95,7 +88,7 @@ function getHeadsets($) { return headsets; } -async function getGameDataFromSearchResult(searchResult) { +function getGameDataFromSearchResult(searchResult) { let $ = _cheerio.load(searchResult); let gameData = {