Download advanced data as CSV (#14)

* Download data as CSV * Refactoring * Add countdown data * Create regex utils, string utils * Fix games with demo
KevinNovak · Nov 22, 2019 · 6053c8e · 6053c8e
1 parent 0faffd1
commit 6053c8e
Show file tree

Hide file tree

Showing 6 changed files with 288 additions and 134 deletions.
diff --git a/app.js b/app.js
@@ -12,14 +12,8 @@ function main() {
 
     _app.post('/api/app-scrape', async (req, res) => {
         let appUrl = req.body.url;
-        let gamePageData = await _steamScraper.getAppPageData(appUrl);
-        res.json(gamePageData);
-    });
-
-    _app.post('/api/headset-scrape', async (req, res) => {
-        let searchUrl = req.body.url;
-        let searchPageData = await _steamScraper.getHeadsetsFromAppPage(searchUrl);
-        res.json(searchPageData);
+        let appPageData = await _steamScraper.getAppPageData(appUrl);
+        res.json(appPageData);
     });
 
     _app.post('/api/search-scrape', async (req, res) => {
@@ -28,6 +22,12 @@ function main() {
         res.json(searchPageData);
     });
 
+    _app.post('/api/search-app-scrape', async (req, res) => {
+        let appUrl = req.body.url;
+        let appPageData = await _steamScraper.getSearchAppPageData(appUrl);
+        res.json(appPageData);
+    });
+
     _app.listen(PORT, () => {
         console.log(`App listening on port ${PORT}!`);
     });

diff --git a/public/index.html b/public/index.html
@@ -264,6 +264,8 @@ <h5>
       integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM"
       crossorigin="anonymous"
     ></script>
+
+    <script src="https://cdn.jsdelivr.net/npm/json2csv"></script>
     <script src="./scripts/index.js"></script>
   </body>
 </html>
diff --git a/public/scripts/index.js b/public/scripts/index.js
@@ -6,7 +6,9 @@ const PERCENT_NUMBER_REGEX = /(\d+)%/;
 const NEW_LINE = '&#10';
 const MAX_PAGES = 100;
 
-const headsetAliases = {
+const BUNDLE_PREFIX = "**Bundle** - ";
+
+const HEADSET_ALIASES = {
     'Valve Index': {
         shortName: 'Index',
         abbreviation: 'I'
@@ -29,6 +31,10 @@ const headsetAliases = {
     }
 }
 
+let cache = {
+    searchData: []
+}
+
 async function retrieveSteamAppTitle() {
     let retrievePageButton = document.getElementById('retrieve-steam-app-title');
     let pageResultsDiv = document.getElementById('page-results');
@@ -51,24 +57,17 @@ async function retrieveSteamAppTitle() {
     try {
         let appData = await post('./api/app-scrape', content);
 
-        let discounted = appData.discounted;
-        let isVr = appData.headsets.length > 0;
-
-        let link = document.createElement('a');
-        if (isVr) {
+        let text = "";
+        if (appData.headsets.length > 0) {
             let platforms = getPlatformText(appData.headsets);
-            if (discounted) {
-                link.innerText = `[${platforms}] ${appData.title} (${appData.price} / ${appData.percentOff} off)`;
-            } else {
-                link.innerText = `[${platforms}] ${appData.title} (${appData.price})`;
-            }
-        } else {
-            if (discounted) {
-                link.innerText = `${appData.title} (${appData.price} / ${appData.percentOff} off)`;
-            } else {
-                link.innerText = `${appData.title} (${appData.price})`;
-            }
+            text += `[${platforms}] `
         }
+        text += `${appData.title} `
+        let priceTag = appData.percentOff ? `(${appData.price} / ${appData.percentOff} off)` : `(${appData.price})`;
+        text += `${priceTag}`
+
+        let link = document.createElement('a');
+        link.innerText = text;
         link.href = appData.link;
         link.target = '_blank';
         link.style.display = 'inline';
@@ -126,12 +125,20 @@ async function retrieveSteamSearchTable() {
         for (let [index, app] of searchData.entries()) {
             let itemNumber = index + 1;
             searchResultsDiv.innerHTML = `Retrieving result ${itemNumber} of ${searchData.length}...`;
-            app.headsets = [];
             if (app.type == "APP") {
                 let content = {
                     url: app.link
                 };
-                app.headsets = await post('./api/headset-scrape', content);
+
+                let appData = await post('./api/search-app-scrape', content);
+                app.headsets = appData.headsets || [];
+                app.countdown = appData.countdown || { text: "", time: 0 };
+            } else {
+                app.headsets = [];
+                app.countdown = {
+                    text: "",
+                    time: 0
+                }
             }
         }
 
@@ -143,8 +150,17 @@ async function retrieveSteamSearchTable() {
         textArea.readOnly = true;
         textArea.innerHTML = text;
 
+        let csv = json2csv.parse(cache.searchData);
+
+        let downloadLink = document.createElement('a');
+        downloadLink.href = 'data:text/csv;charset=utf-8,' + encodeURI(csv);
+        downloadLink.target = '_blank';
+        downloadLink.innerHTML = 'Download Raw Data as CSV';
+        downloadLink.download = `steam-data-${getFormattedTime()}.csv`;
+
         searchResultsDiv.innerHTML = "";
         searchResultsDiv.appendChild(textArea);
+        searchResultsDiv.appendChild(downloadLink);
     } catch (error) {
         console.error(error);
         searchResultsDiv.innerHTML = "No results.";
@@ -153,6 +169,55 @@ async function retrieveSteamSearchTable() {
     retrieveSearchButton.disabled = false;
 }
 
+function getFormattedTime() {
+    let today = new Date();
+    let y = today.getFullYear();
+    // JavaScript months are 0-based.
+    let m = today.getMonth() + 1;
+    let d = today.getDate();
+    let h = today.getHours();
+    let mi = today.getMinutes();
+    let s = today.getSeconds();
+    return y + "-" + m + "-" + d + "-" + h + "-" + mi + "-" + s;
+}
+
+function formatAppData(app) {
+    let formattedData = {
+        type: "",
+        platform: "",
+        platformAbbreviated: "",
+        title: "",
+        titleLink: "",
+        link: "",
+        price: "",
+        originalPrice: "",
+        percentOff: "",
+        countdownText: "",
+        countdownTime: 0,
+        reviews: "",
+        reviewsCount: ""
+    }
+
+    formattedData.type = app.type;
+    formattedData.platform = app.headsets.join(', ');
+    formattedData.platformAbbreviated = app.headsets.map(platform => getHeadsetAbbreviation(platform)).join('/');
+    formattedData.title = app.title;
+
+    let titlePrefix = app.type == "BUNDLE" ? BUNDLE_PREFIX : "";
+    formattedData.titleLink = `${titlePrefix}[${escapePipes(app.title)}](${app.link})`;
+
+    formattedData.link = app.link;
+    formattedData.price = extractNumberFromPrice(app.price) || app.price;
+    formattedData.originalPrice = extractNumberFromPrice(app.originalPrice) || app.price;
+    formattedData.percentOff = extractNumberFromPercent(app.percentOff) || app.percentOff;
+    formattedData.countdownText = app.countdown.text;
+    formattedData.countdownTime = app.countdown.time;
+    formattedData.reviews = extractNumberFromPercent(app.reviewsPercent) || app.reviewsPercent;
+    formattedData.reviewsCount = app.reviewsCount;
+
+    return formattedData;
+}
+
 async function retrieveSearchPageData(steamSearchUrl, pageNumber) {
     let content = {
         url: `${steamSearchUrl}`
@@ -168,20 +233,16 @@ function createMarkdownTable(searchData) {
     let divider = '| :- | :- | -: | -: | -: | -: |';
     let result = header + NEW_LINE + divider + NEW_LINE;
 
-    for (let app of searchData) {
-        let platform = app.headsets.map(platform => getHeadsetAbbreviation(platform)).join('/');
-        let title = escapePipes(app.title);
-        let link = app.link;
-        let price = extractNumberFromPrice(app.price) || app.price || "";
-        let percentOff = extractNumberFromPercent(app.percentOff) || app.percentOff || "";
-        let reviews = extractNumberFromPercent(app.reviewsPercent) || app.reviewsPercent || "";
-        let reviewsCount = app.reviewsCount || "";
+    let formattedData = [];
 
-        let bundlePrefix = app.type == "BUNDLE" ? "**Bundle** - " : "";
-
-        result += `| ${platform} | ${bundlePrefix}[${title}](${link}) | ${price} | ${percentOff} | ${reviews} | ${reviewsCount} |` + NEW_LINE;
+    for (let app of searchData) {
+        let formatted = formatAppData(app);
+        result += `| ${formatted.platformAbbreviated} | ${formatted.titleLink} | ${formatted.price} | ${formatted.percentOff} | ${formatted.reviews} | ${formatted.reviewsCount} |` + NEW_LINE;
+        formattedData.push(formatted);
     }
 
+    cache.searchData = formattedData;
+
     return result;
 }
 
@@ -225,7 +286,7 @@ function getPlatformText(platforms) {
 }
 
 function getHeadsetshortName(headsetName) {
-    let headsetAlias = headsetAliases[headsetName];
+    let headsetAlias = HEADSET_ALIASES[headsetName];
     if (headsetAlias) {
         return headsetAlias.shortName;
     } else {
@@ -234,7 +295,7 @@ function getHeadsetshortName(headsetName) {
 }
 
 function getHeadsetAbbreviation(headsetName) {
-    let headsetAlias = headsetAliases[headsetName];
+    let headsetAlias = HEADSET_ALIASES[headsetName];
     if (headsetAlias) {
         return headsetAlias.abbreviation;
     } else {