script to grab alexa top domains and stats

get_alexa_sites.py

import argparse
import requests
import pandas as pd
import datetime as dt
from bs4 import BeautifulSoup

def get_site_divs(category):
    alexa_base_url = "https://www.alexa.com/topsites/category/Top/"
    if not category:
        # default to global top sites
        alexa_url = "https://www.alexa.com/topsites"
    else:
        alexa_url = "{base}{cat}".format(base=alexa_base_url, cat=category.replace(" ", "_").title())    
    alexa_html = requests.get(alexa_url).content
    soup = BeautifulSoup(alexa_html, 'html.parser')
    return [div for div in soup.find_all("div") if div.get("class") and div.get("class") == ['tr', 'site-listing']]

def parse_site_divs(site_divs):
    data = []
    for i, site in enumerate(site_divs):
        rank = [str(i + 1)]
        site_name = [site.find("p").text.replace("\n", "")]
        site_metric = [div.find("p").text for div in site.find_all("div") if div.get("class") and div.get("class") == ['td', 'right']]
        data += [rank + site_name + site_metric]
    return pd.DataFrame(data, columns=["Rank",
                                       "Site", 
                                       "Daily Time on Site", 
                                       "Daily Pageviews per Visitor", 
                                       "% of Traffic From Search", 
                                       "Total Sites Linking In"])


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Fetch Top Alexa Sites. Return as Pandas DF.")
    parser.add_argument("-c", 
                        dest="c", 
                        type=str, 
                        action="store", 
                        default=None,
                        help="Alexa Site Category (e.g. \'News\' or \'Sports\').\n\
                        Supports subcategories (e.g. \'Sports/Football\')")
    args = parser.parse_args()
    category = args.c
    
    now = dt.datetime.strftime(dt.datetime.now(), "%Y%m%d_%H%M")
    parse_site_divs(get_site_divs(category)).to_csv("{}_{}.csv".format(
        now, 
        category.lower().replace("/", "_") if category else "global_top"),
                                                    header=True,
                                                    index=False
                                                   )
    

Usage:

python3 alexa.py -c Science/Technology
python3 alexa.py -c News