Real-time Scraping - Simple Script

main.py

import requests
from bs4 import BeautifulSoup

r = requests.get('https://store.steampowered.com/search/?term=The+witcher')

soup = BeautifulSoup(r.text, 'html.parser')

resultsRow = soup.find_all('a', {'class': 'search_result_row'})

results = []

for resultRow in resultsRow:
    gameURL = resultRow.get('href')
    title = resultRow.find('span', {'class': 'title'}).text
    releaseDate = resultRow.find('div', {'class': 'search_released'}).text
    imgURL = resultRow.select('div.search_capsule img')[0].get('src')
    price = None
    discountedPrice = None

    # The price is a bit more tricky to get since it is not always there
    # Basically we're checking if it is there, then we get rid of the
    # whitespaces.
    # Then we look for a discounted price, if it exists, we get the full
    # text of the div.search_price selector. This will return the full
    # price appended at the discounted price.
    # We then simply replace the price by an empty string to get only
    # the value of the discounted price.

    if (resultRow.select('div.search_price span strike')):
        price = resultRow.select('div.search_price span strike')[
            0].text.strip(' \t\n\r')
        if (resultRow.select('div.search_price')):
            rawDiscountPrice = resultRow.select(
                'div.search_price')[0].text.strip(' \t\n\r')
            discountedPrice = rawDiscountPrice.replace(price, '')

    # Once formatted, the data are then appended to the results list
    results.append({
        'gameURL': gameURL,
        'title': title,
        'releaseDate': releaseDate,
        'imgURL': imgURL,
        'price': price,
        'discountedPrice': discountedPrice
    })

print(results)