Quickpaste Fetcher

fetcher.py

"""
Ultra-simple fetcher for all pastes at qp.mniip.com.
It really just tries out all combinations from "00" until "zz"...
"""
 
DELAY = 3 # seconds
START = 0 # "00"
END = 1295 # "zz"
OUTPUT_DIR = "./result" # output folder
OVERWRITE = False # fetch again even if already downloaded
IGNORE_404d = True # ignore pastes that 404'd before
ADD_404d = True # add 404d pastes to the fail list
 
import sys
import os
import time
import atexit
import urllib2
 
 
def encode(number):
    alphabet = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'.lower()
 
    base36 = ''
    while number:
        number, i = divmod(number, 36)
        base36 = alphabet[i] + base36
 
    return base36 or alphabet[0]
 
if __name__ == "__main__":
    excpath = os.path.dirname(unicode(sys.argv[0]))
    os.chdir(excpath)
    
    with open("404d.txt", "rt") as reader:
        p404d = reader.read().split(" ")
    
    def f(t):
        with open("404d.txt", "wt") as writer:
            writer.write(" ".join(t))
    #atexit.register(f, p404d)
 
    try:
        for i in xrange(START, END + 1):
            paste = encode(i)
            path = os.path.join(OUTPUT_DIR, encode(i) + ".txt")
 
            if not OVERWRITE and os.path.isfile(path):
                continue
            if IGNORE_404d and paste in p404d:
                continue
 
            try:
                print("-> Downloading %s" % paste)
                response = urllib2.urlopen("http://qp.mniip.com/p/%s" % paste)
 
                print("<- OK, writing to %s" % path)
                with open(path, "wb") as writer:
                    writer.write(response.read())
            except urllib2.HTTPError as e:
                print("<- Nope, %s" % e)
                if ADD_404d:
                    p404d.append(paste)
 
            print("-- waiting %s seconds..." % DELAY)
            time.sleep(DELAY)
            f(p404d) # fuck that
        print("! All done!")
 
    except BaseException as e:
        print("! Stopping because of unhandled exception: %s" % e)