4chan thread archiver. Old project; not maintained.

4chan_archive.py

#!/usr/bin/env python3

from urllib.request import urlopen, urlretrieve
from urllib.error import HTTPError
import json
import os
import sys
import time

def update(thread):
    try:
        old_thread = json.load(open('thread.json'))
    except:
        json.dump(thread, open('thread.json', 'w'))
        return len(thread['posts'])
    have = set(p['no'] for p in old_thread['posts'])
    new_posts = 0
    for p in thread['posts']:
        if p['no'] in have:
            continue
        old_thread['posts'].append(p)
        new_posts += 1
    if new_posts:
        json.dump(thread, open('thread.json', 'w'))
    return new_posts

def download(board, thread_id):
    thread = json.load(urlopen('https://a.4cdn.org/{}/thread/{}.json'.format(board, thread_id)))
    new_posts = update(thread)
    if new_posts:
        print('{} new post{}!'.format(new_posts, '' if new_posts == 1 else 's'))
    else:
        print('no new posts :(')
    for post in thread['posts']:
        if 'filename' not in post:
            continue
        fname = '{}{}'.format(post['tim'], post['ext'])
        if os.path.isfile(fname):
            continue
        print('fetching: {}'.format(fname))
        urlretrieve('https://i.4cdn.org/{}/{}'.format(board, fname), fname)
    if thread["posts"][0].get('archived'):
        print("Thread was archived, we're done here")
        sys.exit(0)

def parse_arg(arg):
    chunks = arg.split('/')
    if len(chunks) != 2:
        return None
    return chunks

board, thread = parse_arg(sys.argv[1])

try:
    os.makedirs(sys.argv[1])
except OSError:
    pass
os.chdir(sys.argv[1])

while True:
    try:
        download(board, thread)
    except HTTPError:
        print('RIP, thread')
        break
    time.sleep(20)

4chan_archivebg.sh

#!/bin/bash

./archive "$@" &>/dev/null & disown