Scrape Medium Earnings

scrape-medium-earnings.py

from seleniumwire import webdriver
import time
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import json

caps = DesiredCapabilities.CHROME
caps['loggingPrefs'] = {'performance': 'ALL'}

options = webdriver.ChromeOptions()
# Go to: chrome://version
# And copy Profile Path
options.add_argument("user-data-dir=/path/to/your/data/dir")

# executable_path will need to point to where you have your Chromedriver executable
driver = webdriver.Chrome(executable_path="./chromedriver", chrome_options=options, desired_capabilities=caps)

driver.get('https://medium.com/me/partner/dashboard')
stat_links = driver.find_elements_by_xpath('//a[text()="View stats"]')

urls = []
posts = []

for link in stat_links:
    url = link.get_attribute('href')
    urls.append(url)

for url in urls:
    post_id = url.replace('https://medium.com/me/stats/post/', '')
    driver.get(url)
    time.sleep(3)

    stats = {}
    for request in driver.requests:
        if request.response and 'graphql' in request.path:
            body = request.response.body.decode('utf-8')
            if 'dailyStats' in body:
                posts.append(json.loads(body))

with open('posts.json', 'w') as f:
    f.write(json.dumps(posts))