Download all images in 4chan thread(s)

fetchchan.sh

#!/usr/bin/bash

# Usage: fetchchan https://boards.4chan.org/{board}/thread/{thread_id} [...]
#  or fetchchan -U in a directory housing a previous fetchchan to download only new images since last fetch

if [[ "$1" == "-U" ]]; then
	urls="$(find . -name .url -type f -exec cat {} \;)"
	set -- $urls
fi

for url in $*; do
		[[ "$url" =~ 4chan.org/(.+)/thread/(.+)/? ]] 

		board="${BASH_REMATCH[1]}"
		thread="${BASH_REMATCH[2]}"
		echo "Downloading $thread from $board"
		mkdir -p ./${thread}
		(
			cd ./${thread}
			<<<"$url" tee .url
			curl -s https://a.4cdn.org/$board/thread/${thread}.json \
			| jq '.posts[] | select(.filename)? | [ .tim,.ext ]' \
			| jq -r 'join("")' \
			| xargs -I {} -n 1 -P 8 \
					sh -c "\
						if ! [ -f .{} ]; then \
							wget -q --show-progress --no-clobber \"https://i.4cdn.org/$board/{}\" \
							&& touch .{}; \
						fi"
		) &
done
wait