Grab u18chan /gfur/

grab-u18.bash

#!/bin/bash

# This script needs httpie and pup
# https://github.com/jkbrzt/httpie
# https://github.com/ericchiang/pup

SITE="http://web.archive.org"
PLACE="/web/20151208204911/https://u18chan.com/board/u18chan/gfur"
OUTDIR="gfur"

get_images() {
    local out="$OUTDIR/$1"
    mkdir -p "$out"

    local images=`http --follow get $SITE$PLACE/topic/$1 | pup '.FileDetails a attr{href}'`
    for i in $images; do
        wget "$SITE$i" -P "$out"
    done
}

get_thread() {
    local threads=`http --follow get $SITE$PLACE/$1 | pup '.reply_link attr{href}' | sed -re 's:.*/([0-9]+):\1:g'`
    for t in $threads; do
        get_images $t
        sleep 1
    done
}

for p in `seq 0 65`; do
    get_thread $p
    sleep 2
done