gistfile1.txt

import requests, json, hashlib, re, urllib.request, sqlite3
from multiprocessing import Pool

con = sqlite3.connect('db/memory.db')
cur = con.cursor()

def find_between( s, first, last ):
    try:
        start = s.index( first ) + len( first )
        end = s.index( last, start )
        return s[start:end]
    except ValueError:
        return ""

def checkDefaultAvatar(user):
    for row in cur.execute('SELECT twitter_avatar FROM users WHERE user_name="{0}" LIMIT 1'.format(user)):
        twitterAvatar = row[0]
    if "default_profile_images" in twitterAvatar:
        return True
    else:
        return False


def checkAvatar(path):
    url = "https://8.azureedge.net/img?l={0}".format(path)
    r = requests.get(url)
    hash = doMD5(r.text)
    return hash

def getName(data):
    result = re.search('([0-9]+)\_(.*)\.jpg', data["name"])
    name = result.group(2)    
    return name

def saveImage(url, name):
    url = "https://8.azureedge.net/img?l={0}".format(url)
    urllib.request.urlretrieve(url, "imgs/{0}.jpg".format(name))

def getTile(tileCoords):
    try:
        x = tileCoords["x"]
        y = tileCoords["y"]
        url = "https://mosaically.com/slmake/gettileinfoatxy/?MosaicId=fd40c771-f976-4bf7-96b8-80379da82333&selectedRow=-1&x={0}&y={1}".format(x,y)
        r = requests.get(url)
        data = json.loads(r.text)
        # name = data["name"].split("_")[1]
        name = getName(data)
        thumb = data["thumb"]
        thumbMD5 = checkAvatar(thumb)
        if checkDefaultAvatar(name):
            if thumbMD5 != "611f87b37c4cbcb8143ccbcd6207277a":
                saveImage(thumb, name)
                print(x,y,name,thumb,thumbMD5)
    except Exception as e:
        print('exception: {0}  - {1}, {2}'.format(e, x, y))
        return

    print('finished: {0}, {1}'.format(x,y))


def doMD5(input):
    m = hashlib.md5()
    m.update(input.encode("utf-8"))
    return m.hexdigest()

toCheck = []

for x in range(0, 182):
    for y in range (0, 273):
        toCheck.append({"x":x,"y":y})

if __name__ == '__main__':
    pool = Pool(processes=10)
    pool.map(getTile, toCheck)