games-express-api/scripts/scraper.js

176 lines
7.5 KiB
JavaScript
Raw Normal View History

2024-09-12 15:48:27 -04:00
import {parse} from "node-html-parser";
const steamScraper = async (resData) => {
const {_id, steamId, series, wine, intel} = resData
const {playStatus, soundtrack} = resData.accessedBy[0]
if (steamId === null || steamId === '') {
return 'Please enter a valid Steam Id'
}
const steamApiUrl = `https://store.steampowered.com/api/appdetails/?appids=${steamId}`
const steamRatingApiUrl = `https://store.steampowered.com/appreviews/${steamId}?json=1`
const steamWebUrl = `https://store.steampowered.com/app/${steamId}`
try {
const apiResponse = await fetch(steamApiUrl, {
headers: {
Accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US',
Connection: 'keep-alive',
Host:
'store.steampowered.com',
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
},
}),
steamRatingApiResponse = await fetch(steamRatingApiUrl, {
headers: {
Accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US',
Connection: 'keep-alive',
Cookie: 'birthtime=470682001;path=/;domain=store.steampowered.com',
Host:
'store.steampowered.com',
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
},
}),
webResponse = await fetch(steamWebUrl, {
headers: {
Accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US',
Connection: 'keep-alive',
Cookie: 'birthtime=470682001;path=/;domain=store.steampowered.com',
Host:
'store.steampowered.com',
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
},
}),
apiAnswer = await apiResponse.json(),
steamRatingApiAnswer = await steamRatingApiResponse.json(),
webAnswer = await webResponse.text(),
success = apiAnswer[steamId]['success']
if (success === true) {
const data = apiAnswer[steamId]['data']
const steamRating = steamRatingApiAnswer['success'] === 1 ?
steamRatingApiAnswer['query_summary']['review_score'] :
0
const webData = parse(webAnswer)
const screenshots = data.screenshots?.map((screenshot) => ({
id: screenshot.id,
full: screenshot.path_full,
thumbnail: screenshot.path_thumbnail,
})) || [],
steamGenres = data.genres?.map((genre) => genre.description) || [],
tags = webData.querySelectorAll('.app_tag').map(x => {
let tag = x.rawText.trim() !== '+' && x.rawText.trim()
if (typeof tag == 'string' && tag.includes("&")) tag = tag.replace("&", "&")
return tag
}),
manualGenres = resData.genre || [],
genre = [...new Set([...steamGenres, ...manualGenres, ...tags])].filter(x => x !== "" & x !== false),
filterController = data.categories
? data.categories.filter((x) => {
if (x.id === 28 || x.id === 18) {
return x
}
})
: [],
controller =
filterController.length === 1
? filterController[0].description
.split(' ')
.map(word => word[0].toUpperCase() + word.substring(1).toLowerCase())
.join(' ')
: 'No Controller Support',
store = resData.accessedBy[0].store.length > 0 ? resData.accessedBy[0].store : ['Steam'],
createdBy = resData.createdBy || '',
lastModifiedBy = resData.lastModifiedBy || '',
user = resData.accessedBy[0].user || '',
reviews = resData.reviews || '',
rating = resData.rating || 0,
json = {
_id,
steamId,
title: data.name,
series,
frontImage: data.header_image,
screenshots,
genre,
os: data.platforms,
wine,
controller,
developer: data.developers,
publisher: data.publishers,
releaseDate: data.release_date.date,
shortDesc: data.short_description,
reviews,
summary: data.about_the_game,
steamRating,
intel,
systemRequirements: {
windows: {
minimum: '',
recommended: '',
},
mac: {
minimum: '',
recommended: '',
},
linux: {
minimum: '',
recommended: '',
},
},
createdBy,
lastModifiedBy,
accessedBy: [
{
user,
store,
playStatus,
soundtrack,
},
],
rating,
}
if (data.platforms.windows) {
json.systemRequirements.windows.minimum =
data.pc_requirements.minimum || ''
json.systemRequirements.windows.recommended =
data.pc_requirements.recommended || ''
}
if (data.platforms.mac) {
json.systemRequirements.mac.minimum =
data.mac_requirements.minimum || ''
json.systemRequirements.mac.recommended =
data.mac_requirements.recommended || ''
}
if (data.platforms.linux) {
json.systemRequirements.linux.minimum =
data.linux_requirements.minimum || ''
json.systemRequirements.linux.recommended =
data.linux_requirements.recommended || ''
}
return json
} else {
return success
}
} catch (error) {
console.error(error)
return error
}
}
export default steamScraper