diff --git a/app.js b/app.js index 77e7060..f03314f 100644 --- a/app.js +++ b/app.js @@ -16,6 +16,12 @@ function main() { res.json(gamePageData); }); + _app.post('/api/headset-scrape', async (req, res) => { + let searchUrl = req.body.url; + let searchPageData = await _steamScraper.getHeadsetsFromAppPage(searchUrl); + res.json(searchPageData); + }); + _app.post('/api/search-scrape', async (req, res) => { let searchUrl = req.body.url; let searchPageData = await _steamScraper.getSearchPageData(searchUrl); diff --git a/package-lock.json b/package-lock.json index 7e25e90..bbc55ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2419,9 +2419,9 @@ "dev": true }, "is-retry-allowed": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.1.0.tgz", - "integrity": "sha1-EaBgVotnM5REAz0BJaYaINVk+zQ=", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", + "integrity": "sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==", "dev": true }, "is-stream": { @@ -2779,18 +2779,18 @@ "integrity": "sha1-ICl+idhvb2QA8lDZ9Pa0wZRfzTU=" }, "nodemon": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-1.19.2.tgz", - "integrity": "sha512-hRLYaw5Ihyw9zK7NF+9EUzVyS6Cvgc14yh8CAYr38tPxJa6UrOxwAQ351GwrgoanHCF0FalQFn6w5eoX/LGdJw==", + "version": "1.19.4", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-1.19.4.tgz", + "integrity": "sha512-VGPaqQBNk193lrJFotBU8nvWZPqEZY2eIzymy2jjY0fJ9qIsxA0sxQ8ATPl0gZC645gijYEc1jtZvpS8QWzJGQ==", "dev": true, "requires": { - "chokidar": "^2.1.5", - "debug": "^3.1.0", + "chokidar": "^2.1.8", + "debug": "^3.2.6", "ignore-by-default": "^1.0.1", "minimatch": "^3.0.4", - "pstree.remy": "^1.1.6", - "semver": "^5.5.0", - "supports-color": "^5.2.0", + "pstree.remy": "^1.1.7", + "semver": "^5.7.1", + "supports-color": "^5.5.0", "touch": "^3.1.0", "undefsafe": "^2.0.2", "update-notifier": "^2.5.0" diff --git a/package.json b/package.json index fe042c2..3c64573 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "request-promise": "4.2.4" }, "devDependencies": { - "nodemon": "^1.19.2" + "nodemon": "^1.19.4" }, "homepage": "https://github.com/KevinNovak/vrgamedeals-Tools#readme", "repository": { diff --git a/public/scripts/index.js b/public/scripts/index.js index 35890c9..712d5fa 100644 --- a/public/scripts/index.js +++ b/public/scripts/index.js @@ -4,7 +4,7 @@ const PRICE_NUMBER_REGEX = /\$(\d+\.\d{2})/; const PERCENT_NUMBER_REGEX = /(\d+)%/; const NEW_LINE = ' '; -const MAX_PAGES = 20; +const MAX_PAGES = 100; const headsetAliases = { 'Valve Index': { @@ -105,21 +105,15 @@ async function retrieveSteamSearchTable() { let searchData = []; if (searchAllPages) { for (let i = 1; i <= MAX_PAGES; i++) { - let content = { - url: `${steamSearchUrl}&page=${i}` - }; - searchResultsDiv.innerHTML = `Retrieving page ${i}...`; - let searchPageData = await post('./api/search-scrape', content); - if (searchPageData.length < 1) { - break; + let searchPageData = await retrieveSearchPageData(searchResultsDiv, steamSearchUrl, i); + if (searchPageData.length < 1) { + break; } searchData.push(...searchPageData); } } else { - let content = { - url: steamSearchUrl - }; - searchData = await post('./api/search-scrape', content); + let searchPageData = await retrieveSearchPageData(searchResultsDiv, steamSearchUrl); + searchData.push(...searchPageData); } if (!searchData || searchData.length < 1) { @@ -145,6 +139,25 @@ async function retrieveSteamSearchTable() { retrieveSearchButton.disabled = false; } +async function retrieveSearchPageData(searchResultsDiv, steamSearchUrl, pageNumber = 1) { + let content = { + url: `${steamSearchUrl}&page=${pageNumber}` + }; + let searchPageData = await post('./api/search-scrape', content); + for (let [index, app] of searchPageData.entries()) { + let itemNumber = index + 1; + searchResultsDiv.innerHTML = `Retrieving page ${pageNumber}, result ${itemNumber} of ${searchPageData.length}...`; + app.headsets = []; + if (app.type == "APP") { + let content = { + url: app.link + }; + app.headsets = await post('./api/headset-scrape', content); + } + } + return searchPageData; +} + function createMarkdownTable(searchData) { let header = '| Platform | Title | Price (USD) | Discount (%) | Rating (%) | Review Count |'; let divider = '| :- | :- | -: | -: | -: | -: |'; diff --git a/services/steam-scraper.js b/services/steam-scraper.js index ef0b07e..313fe72 100644 --- a/services/steam-scraper.js +++ b/services/steam-scraper.js @@ -80,7 +80,7 @@ function getHeadsets($) { return headsets; } -async function getHeadsetsFromLink(link) { +async function getHeadsetsFromAppPage(link) { let pageHtml = await _rp({ url: link }); let $ = _cheerio.load(pageHtml); return getHeadsets($); @@ -98,7 +98,6 @@ async function getGameDataFromSearchResult(searchResult) { let percentOff = ""; let reviewsPercent = ""; let reviewsCount = ""; - let headsets = []; title = $('div.search_name > span.title').text().trim(); link = stripQueryString(searchResult.attribs.href); @@ -125,8 +124,6 @@ async function getGameDataFromSearchResult(searchResult) { reviewsPercent = extractPercent(reviewsSummary); reviewsCount = extractReviewsCount(reviewsSummary).replace(/,/g, ''); } - - headsets = await getHeadsetsFromLink(link); } return { @@ -139,7 +136,6 @@ async function getGameDataFromSearchResult(searchResult) { percentOff, reviewsPercent, reviewsCount, - headsets }; } @@ -179,5 +175,6 @@ function getGameDataFromGameElement(gameElement) { module.exports = { getAppPageData, + getHeadsetsFromAppPage, getSearchPageData }; \ No newline at end of file