Skip to content

Instantly share code, notes, and snippets.

What would you like to do?
from bs4 import BeautifulSoup as bs
import re
import requests
import pandas as pd
url_list_user, userName, userCity, userCountry, userMug,userPercent,userTrading,userLooking= ([] for i in range(8))
for i in range(1,438):
url_list_user.append('' + str(i))
for url in url_list_user:
raw = bs(requests.get(url).text, "lxml")
titleDiv = raw.findAll("div", { "class" : "mugTitle" })
collection = raw.findAll(text=' of 5329 ')
looking = raw.findAll(text='Looking For:')
userName.append([tag.a.string for tag in titleDiv])
for tag in titleDiv:
for tag in titleDiv:
userMug.append([tag.findPreviousSibling('a').text for tag in collection])
userPercent.append([tag.findNextSibling('small').text for tag in collection])
userTrading.append([tag.findNext('a').text for tag in collection])
userLooking.append([tag.findNext('a').text for tag in looking])
userName = [y for x in userName for y in x]
userMug = [y for x in userMug for y in x]
userPercent = [y for x in userPercent for y in x]
userTrading = [y for x in userTrading for y in x]
userLooking = [y for x in userLooking for y in x]
user = {
'userName': userName,
'City': userCity,
'Country': userCountry,
'MugsOwned': userMug,
'PercentageOwned': userPercent,
'Trading': userTrading,
'Looking': userLooking,
userdf = pd.DataFrame(user)
userdf.to_csv('user.csv', sep='\t', encoding='utf-8')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.