Skip to content

Instantly share code, notes, and snippets.

@ziot
Created May 5, 2021 05:25
Show Gist options
  • Save ziot/c2c681cc6424c62a2b9ebfa506f1eadb to your computer and use it in GitHub Desktop.
Save ziot/c2c681cc6424c62a2b9ebfa506f1eadb to your computer and use it in GitHub Desktop.
import requests, json, hashlib, re, urllib.request, sqlite3
from multiprocessing import Pool
con = sqlite3.connect('db/memory.db')
cur = con.cursor()
def find_between( s, first, last ):
try:
start = s.index( first ) + len( first )
end = s.index( last, start )
return s[start:end]
except ValueError:
return ""
def checkDefaultAvatar(user):
for row in cur.execute('SELECT twitter_avatar FROM users WHERE user_name="{0}" LIMIT 1'.format(user)):
twitterAvatar = row[0]
if "default_profile_images" in twitterAvatar:
return True
else:
return False
def checkAvatar(path):
url = "https://8.azureedge.net/img?l={0}".format(path)
r = requests.get(url)
hash = doMD5(r.text)
return hash
def getName(data):
result = re.search('([0-9]+)\_(.*)\.jpg', data["name"])
name = result.group(2)
return name
def saveImage(url, name):
url = "https://8.azureedge.net/img?l={0}".format(url)
urllib.request.urlretrieve(url, "imgs/{0}.jpg".format(name))
def getTile(tileCoords):
try:
x = tileCoords["x"]
y = tileCoords["y"]
url = "https://mosaically.com/slmake/gettileinfoatxy/?MosaicId=fd40c771-f976-4bf7-96b8-80379da82333&selectedRow=-1&x={0}&y={1}".format(x,y)
r = requests.get(url)
data = json.loads(r.text)
# name = data["name"].split("_")[1]
name = getName(data)
thumb = data["thumb"]
thumbMD5 = checkAvatar(thumb)
if checkDefaultAvatar(name):
if thumbMD5 != "611f87b37c4cbcb8143ccbcd6207277a":
saveImage(thumb, name)
print(x,y,name,thumb,thumbMD5)
except Exception as e:
print('exception: {0} - {1}, {2}'.format(e, x, y))
return
print('finished: {0}, {1}'.format(x,y))
def doMD5(input):
m = hashlib.md5()
m.update(input.encode("utf-8"))
return m.hexdigest()
toCheck = []
for x in range(0, 182):
for y in range (0, 273):
toCheck.append({"x":x,"y":y})
if __name__ == '__main__':
pool = Pool(processes=10)
pool.map(getTile, toCheck)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment