4chanscraper.py

#!/usr/bin/env python3
import urllib.request
import json
import sys
import time

def get_url(url):
    opener = urllib.request.build_opener()
    opener.addheaders = [('User-agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36')]
    return opener.open(url)

def get_json(url):
    with get_url(url) as f:
        return json.loads(f.read().decode("utf-8"))

def save_file(board, f, nonce):
    if ("tim" in f) and ("filename" in f) and ("ext" in f):
        time.sleep(1)
        print(f["tim"],f["filename"],f["ext"])
        with get_url("http://i.4cdn.org/{0}/{1}{2}".format(board, f["tim"],f["ext"])) as u:
            with open("{0:07}-{1}-{2}{3}".format(nonce,f["tim"],f["filename"],f["ext"]),"wb") as f:
                f.write(u.read())

def main():
    """example: $ ./betterthancamo.py wsg 1072213"""
    board = sys.argv[1]
    number = sys.argv[2]
    thread = get_json("http://a.4cdn.org/{0}/thread/{1}.json".format(board,number))
    nonce = 0
    for post in thread["posts"]:
        nonce += 1
        save_file(board, post, nonce)

if __name__ == "__main__":
    main()