Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Download advanced data as CSV #14

Merged
merged 22 commits into from
Nov 22, 2019
Merged

Download advanced data as CSV #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
83e0c80
Extract countdown text and time from app page
KevinNovak Nov 13, 2019
4c1914a
Refactor
KevinNovak Nov 13, 2019
d227bdb
Fix UI for headsets
KevinNovak Nov 13, 2019
2e760d6
Add countdown data to UI
KevinNovak Nov 13, 2019
d0befda
Move formatting to new method
KevinNovak Nov 13, 2019
b6bba59
Add extra data
KevinNovak Nov 13, 2019
218f4a2
Add countdown text and time
KevinNovak Nov 13, 2019
e4762ad
Fix constants
KevinNovak Nov 13, 2019
5f66c85
Create regex utils
KevinNovak Nov 22, 2019
a0cacbd
Consolidate getting first game
KevinNovak Nov 22, 2019
d382aeb
Add string-utils, reorder methods
KevinNovak Nov 22, 2019
321b21e
Refactoring
KevinNovak Nov 22, 2019
0c026cb
Reordering
KevinNovak Nov 22, 2019
4213103
App page also gets countdown
KevinNovak Nov 22, 2019
9a205a5
Safer property setting
KevinNovak Nov 22, 2019
6e9840b
Safer property setting
KevinNovak Nov 22, 2019
5480dd8
Fix games with demo
KevinNovak Nov 22, 2019
c9ace25
Reordering
KevinNovak Nov 22, 2019
134dc25
Refactoring
KevinNovak Nov 22, 2019
1746009
Better way to build link text
KevinNovak Nov 22, 2019
c9ea403
Refactoring
KevinNovak Nov 22, 2019
da024d6
Download data as CSV
KevinNovak Nov 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions app.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,8 @@ function main() {

_app.post('/api/app-scrape', async (req, res) => {
let appUrl = req.body.url;
let gamePageData = await _steamScraper.getAppPageData(appUrl);
res.json(gamePageData);
});

_app.post('/api/headset-scrape', async (req, res) => {
let searchUrl = req.body.url;
let searchPageData = await _steamScraper.getHeadsetsFromAppPage(searchUrl);
res.json(searchPageData);
let appPageData = await _steamScraper.getAppPageData(appUrl);
res.json(appPageData);
});

_app.post('/api/search-scrape', async (req, res) => {
Expand All @@ -28,6 +22,12 @@ function main() {
res.json(searchPageData);
});

_app.post('/api/search-app-scrape', async (req, res) => {
let appUrl = req.body.url;
let appPageData = await _steamScraper.getSearchAppPageData(appUrl);
res.json(appPageData);
});

_app.listen(PORT, () => {
console.log(`App listening on port ${PORT}!`);
});
Expand Down
2 changes: 2 additions & 0 deletions public/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,8 @@ <h5>
integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM"
crossorigin="anonymous"
></script>

<script src="https://cdn.jsdelivr.net/npm/json2csv"></script>
<script src="./scripts/index.js"></script>
</body>
</html>
125 changes: 93 additions & 32 deletions public/scripts/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ const PERCENT_NUMBER_REGEX = /(\d+)%/;
const NEW_LINE = '&#10';
const MAX_PAGES = 100;

const headsetAliases = {
const BUNDLE_PREFIX = "**Bundle** - ";

const HEADSET_ALIASES = {
'Valve Index': {
shortName: 'Index',
abbreviation: 'I'
Expand All @@ -29,6 +31,10 @@ const headsetAliases = {
}
}

let cache = {
searchData: []
}

async function retrieveSteamAppTitle() {
let retrievePageButton = document.getElementById('retrieve-steam-app-title');
let pageResultsDiv = document.getElementById('page-results');
Expand All @@ -51,24 +57,17 @@ async function retrieveSteamAppTitle() {
try {
let appData = await post('./api/app-scrape', content);

let discounted = appData.discounted;
let isVr = appData.headsets.length > 0;

let link = document.createElement('a');
if (isVr) {
let text = "";
if (appData.headsets.length > 0) {
let platforms = getPlatformText(appData.headsets);
if (discounted) {
link.innerText = `[${platforms}] ${appData.title} (${appData.price} / ${appData.percentOff} off)`;
} else {
link.innerText = `[${platforms}] ${appData.title} (${appData.price})`;
}
} else {
if (discounted) {
link.innerText = `${appData.title} (${appData.price} / ${appData.percentOff} off)`;
} else {
link.innerText = `${appData.title} (${appData.price})`;
}
text += `[${platforms}] `
}
text += `${appData.title} `
let priceTag = appData.percentOff ? `(${appData.price} / ${appData.percentOff} off)` : `(${appData.price})`;
text += `${priceTag}`

let link = document.createElement('a');
link.innerText = text;
link.href = appData.link;
link.target = '_blank';
link.style.display = 'inline';
Expand Down Expand Up @@ -126,12 +125,20 @@ async function retrieveSteamSearchTable() {
for (let [index, app] of searchData.entries()) {
let itemNumber = index + 1;
searchResultsDiv.innerHTML = `Retrieving result ${itemNumber} of ${searchData.length}...`;
app.headsets = [];
if (app.type == "APP") {
let content = {
url: app.link
};
app.headsets = await post('./api/headset-scrape', content);

let appData = await post('./api/search-app-scrape', content);
app.headsets = appData.headsets || [];
app.countdown = appData.countdown || { text: "", time: 0 };
} else {
app.headsets = [];
app.countdown = {
text: "",
time: 0
}
}
}

Expand All @@ -143,8 +150,17 @@ async function retrieveSteamSearchTable() {
textArea.readOnly = true;
textArea.innerHTML = text;

let csv = json2csv.parse(cache.searchData);

let downloadLink = document.createElement('a');
downloadLink.href = 'data:text/csv;charset=utf-8,' + encodeURI(csv);
downloadLink.target = '_blank';
downloadLink.innerHTML = 'Download Raw Data as CSV';
downloadLink.download = `steam-data-${getFormattedTime()}.csv`;

searchResultsDiv.innerHTML = "";
searchResultsDiv.appendChild(textArea);
searchResultsDiv.appendChild(downloadLink);
} catch (error) {
console.error(error);
searchResultsDiv.innerHTML = "No results.";
Expand All @@ -153,6 +169,55 @@ async function retrieveSteamSearchTable() {
retrieveSearchButton.disabled = false;
}

function getFormattedTime() {
let today = new Date();
let y = today.getFullYear();
// JavaScript months are 0-based.
let m = today.getMonth() + 1;
let d = today.getDate();
let h = today.getHours();
let mi = today.getMinutes();
let s = today.getSeconds();
return y + "-" + m + "-" + d + "-" + h + "-" + mi + "-" + s;
}

function formatAppData(app) {
let formattedData = {
type: "",
platform: "",
platformAbbreviated: "",
title: "",
titleLink: "",
link: "",
price: "",
originalPrice: "",
percentOff: "",
countdownText: "",
countdownTime: 0,
reviews: "",
reviewsCount: ""
}

formattedData.type = app.type;
formattedData.platform = app.headsets.join(', ');
formattedData.platformAbbreviated = app.headsets.map(platform => getHeadsetAbbreviation(platform)).join('/');
formattedData.title = app.title;

let titlePrefix = app.type == "BUNDLE" ? BUNDLE_PREFIX : "";
formattedData.titleLink = `${titlePrefix}[${escapePipes(app.title)}](${app.link})`;

formattedData.link = app.link;
formattedData.price = extractNumberFromPrice(app.price) || app.price;
formattedData.originalPrice = extractNumberFromPrice(app.originalPrice) || app.price;
formattedData.percentOff = extractNumberFromPercent(app.percentOff) || app.percentOff;
formattedData.countdownText = app.countdown.text;
formattedData.countdownTime = app.countdown.time;
formattedData.reviews = extractNumberFromPercent(app.reviewsPercent) || app.reviewsPercent;
formattedData.reviewsCount = app.reviewsCount;

return formattedData;
}

async function retrieveSearchPageData(steamSearchUrl, pageNumber) {
let content = {
url: `${steamSearchUrl}`
Expand All @@ -168,20 +233,16 @@ function createMarkdownTable(searchData) {
let divider = '| :- | :- | -: | -: | -: | -: |';
let result = header + NEW_LINE + divider + NEW_LINE;

for (let app of searchData) {
let platform = app.headsets.map(platform => getHeadsetAbbreviation(platform)).join('/');
let title = escapePipes(app.title);
let link = app.link;
let price = extractNumberFromPrice(app.price) || app.price || "";
let percentOff = extractNumberFromPercent(app.percentOff) || app.percentOff || "";
let reviews = extractNumberFromPercent(app.reviewsPercent) || app.reviewsPercent || "";
let reviewsCount = app.reviewsCount || "";
let formattedData = [];

let bundlePrefix = app.type == "BUNDLE" ? "**Bundle** - " : "";

result += `| ${platform} | ${bundlePrefix}[${title}](${link}) | ${price} | ${percentOff} | ${reviews} | ${reviewsCount} |` + NEW_LINE;
for (let app of searchData) {
let formatted = formatAppData(app);
result += `| ${formatted.platformAbbreviated} | ${formatted.titleLink} | ${formatted.price} | ${formatted.percentOff} | ${formatted.reviews} | ${formatted.reviewsCount} |` + NEW_LINE;
formattedData.push(formatted);
}

cache.searchData = formattedData;

return result;
}

Expand Down Expand Up @@ -225,7 +286,7 @@ function getPlatformText(platforms) {
}

function getHeadsetshortName(headsetName) {
let headsetAlias = headsetAliases[headsetName];
let headsetAlias = HEADSET_ALIASES[headsetName];
if (headsetAlias) {
return headsetAlias.shortName;
} else {
Expand All @@ -234,7 +295,7 @@ function getHeadsetshortName(headsetName) {
}

function getHeadsetAbbreviation(headsetName) {
let headsetAlias = headsetAliases[headsetName];
let headsetAlias = HEADSET_ALIASES[headsetName];
if (headsetAlias) {
return headsetAlias.abbreviation;
} else {
Expand Down
Loading