games-express-api/scripts/scraper.js

import {parse} from "node-html-parser";

const steamScraper = async (resData) => {
    const {_id, steamId, series, wine, intel} = resData
    const {playStatus, soundtrack} = resData.accessedBy[0]
    if (steamId === null || steamId === '') {
        return 'Please enter a valid Steam Id'
    }
    const steamApiUrl = `https://store.steampowered.com/api/appdetails/?appids=${steamId}`
    const steamRatingApiUrl = `https://store.steampowered.com/appreviews/${steamId}?json=1`
    const steamWebUrl = `https://store.steampowered.com/app/${steamId}`
    try {
        const apiResponse = await fetch(steamApiUrl, {
                headers: {
                    Accept:
                        'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
                    'Accept-Encoding': 'gzip, deflate, br',
                    'Accept-Language': 'en-US',
                    Connection: 'keep-alive',
                    Host:
                        'store.steampowered.com',
                    'Upgrade-Insecure-Requests': '1',
                    'User-Agent':
                        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
                },
            }),
            steamRatingApiResponse = await fetch(steamRatingApiUrl, {
                headers: {
                    Accept:
                        'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
                    'Accept-Encoding': 'gzip, deflate, br',
                    'Accept-Language': 'en-US',
                    Connection: 'keep-alive',
                    Cookie: 'birthtime=470682001;path=/;domain=store.steampowered.com',
                    Host:
                        'store.steampowered.com',
                    'Upgrade-Insecure-Requests': '1',
                    'User-Agent':
                        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
                },
            }),
            webResponse = await fetch(steamWebUrl, {
                headers: {
                    Accept:
                        'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
                    'Accept-Encoding': 'gzip, deflate, br',
                    'Accept-Language': 'en-US',
                    Connection: 'keep-alive',
                    Cookie: 'birthtime=470682001;path=/;domain=store.steampowered.com',
                    Host:
                        'store.steampowered.com',
                    'Upgrade-Insecure-Requests': '1',
                    'User-Agent':
                        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
                },
            }),
            apiAnswer = await apiResponse.json(),
            steamRatingApiAnswer = await steamRatingApiResponse.json(),
            webAnswer = await webResponse.text(),
            success = apiAnswer[steamId]['success']
        if (success === true) {
            const data = apiAnswer[steamId]['data']
            const steamRating = steamRatingApiAnswer['success'] === 1 ?
                steamRatingApiAnswer['query_summary']['review_score'] :
                0
            const webData = parse(webAnswer)

            const screenshots = data.screenshots?.map((screenshot) => ({
                    id: screenshot.id,
                    full: screenshot.path_full,
                    thumbnail: screenshot.path_thumbnail,
                })) || [],
                steamGenres = data.genres?.map((genre) => genre.description) || [],
                tags = webData.querySelectorAll('.app_tag').map(x => {
                    let tag = x.rawText.trim() !== '+' && x.rawText.trim()
                    if (typeof tag == 'string' && tag.includes("&")) tag = tag.replace("&", "&")
                    return tag
                }),
                manualGenres = resData.genre || [],
                genre = [...new Set([...steamGenres, ...manualGenres, ...tags])].filter(x => x !== "" & x !== false),
                filterController = data.categories
                    ? data.categories.filter((x) => {
                        if (x.id === 28 || x.id === 18) {
                            return x
                        }
                    })
                    : [],
                controller =
                    filterController.length === 1
                        ? filterController[0].description
                            .split(' ')
                            .map(word => word[0].toUpperCase() + word.substring(1).toLowerCase())
                            .join(' ')
                        : 'No Controller Support',
                store = resData.accessedBy[0].store.length > 0 ? resData.accessedBy[0].store : ['Steam'],
                createdBy = resData.createdBy || '',
                lastModifiedBy = resData.lastModifiedBy || '',
                user = resData.accessedBy[0].user || '',
                reviews = resData.reviews || '',
                rating = resData.rating || 0,
                json = {
                    _id,
                    steamId,
                    title: data.name,
                    series,
                    frontImage: data.header_image,
                    screenshots,
                    genre,
                    os: data.platforms,
                    wine,
                    controller,
                    developer: data.developers,
                    publisher: data.publishers,
                    releaseDate: data.release_date.date,
                    shortDesc: data.short_description,
                    reviews,
                    summary: data.about_the_game,
                    steamRating,
                    intel,
                    systemRequirements: {
                        windows: {
                            minimum: '',
                            recommended: '',
                        },
                        mac: {
                            minimum: '',
                            recommended: '',
                        },
                        linux: {
                            minimum: '',
                            recommended: '',
                        },
                    },
                    createdBy,
                    lastModifiedBy,
                    accessedBy: [
                        {
                            user,
                            store,
                            playStatus,
                            soundtrack,
                        },
                    ],
                    rating,
                }

            if (data.platforms.windows) {
                json.systemRequirements.windows.minimum =
                    data.pc_requirements.minimum || ''
                json.systemRequirements.windows.recommended =
                    data.pc_requirements.recommended || ''
            }
            if (data.platforms.mac) {
                json.systemRequirements.mac.minimum =
                    data.mac_requirements.minimum || ''
                json.systemRequirements.mac.recommended =
                    data.mac_requirements.recommended || ''
            }
            if (data.platforms.linux) {
                json.systemRequirements.linux.minimum =
                    data.linux_requirements.minimum || ''
                json.systemRequirements.linux.recommended =
                    data.linux_requirements.recommended || ''
            }
            return json
        } else {
            return success
        }
    } catch (error) {
        console.error(error)
        return error
    }
}

export default steamScraper