actuallyuseddota2scraper.py

from bs4 import BeautifulSoup
#import requests
import csv

import csv

# leaderboard_root_url = 'http://www.dota2.com/leaderboards#'
# regions = ['americas','europe', 'se_asia', 'china']


# page = requests.get(leaderboard_root_url + region).text
soup = BeautifulSoup(open("china-7-28.html"), "lxml")
# print(soup.prettify())


# team_names = soup.find_all("span", {"class": "team_tag"})
# for elem in team_names:
#     print(elem.getText())

    # print(soup.encode("utf-8"))

# output = []

leaderboard_table = soup.select_one("#leaderboard_body")
tr = leaderboard_table.findAll("tr")
for row in tr:
    team_tag = row.find("span", class_= "team_tag")
    if team_tag:
      team_tag_name = team_tag.text
    else:
      team_tag_name = "No registered team"

    player_name = row.find("span", class_="player_name")
    if player_name:
      player_name = player_name.text
    else:
      player_name = "No registered name"


    mmr = row.find_all("td")[-1].text

    # output.append("%s,%s,%s" % (team_tag_name, player_name, mmr))

    print("%s,%s,%s" % (team_tag_name.encode("utf-8"), player_name.encode("utf-8"), mmr.encode("utf-8")))

    # player_name = row.select_one("td:nth-child(2) > span.team_tag")
    # solo_mmr = row.select_one("td:nth-child(3)")
    # print(team_tag, player_name, solo_mmr)


    # for row in leaderboard_table.findChildren():
    #     team_tag = row.select_one("td:nth-child(2) > span.team_tag")
    #     player_name = row.select_one("td:nth-child(2) > span.team_tag")
    #     solo_mmr = row.select_one("td:nth-child(3)")

        # print(team_tag, player_name, solo_mmr)

        # print(td[2][2].text)
        # print(td[2][3].text)
        # print(td[3].text)

        # team_tag = row.find("td:nth-child(2) > span.team_tag")
        # player_name = row.find("td:nth-child(2) > span.team_tag")
        # solo_mmr = row.find("td:nth-child(3)")
        # print(team_tag,player_name,solo_mmr,region)