Download all links from reddit search url

README.md

jq is required

Simply pass in a reddit link and the script will output all links from that search, e.g.:

./search2list 'https://www.reddit.com/r/speedrun/search/?q=site%3Ayoutu&include_over_18=on&restrict_sr=on&t=month&sort=relevance~'

And of course if you would like to output it to a file:

./search2list 'https://www.reddit.com/r/speedrun/search/?q=site%3Ayoutu&include_over_18=on&restrict_sr=on&t=month&sort=relevance~' >> mylist

search2list

#!/usr/bin/env bash

if [ -z "$1" ]; then
    echo "No search URL given"
    exit 1
fi

url="$1"

if echo -n "$url" | grep -q '/search\(/\)\??'; then
    url="$(echo -n "$url" | sed -e 's/\/search\/\??/\/search.json?/g')"
else
    echo "Couldn't modify search term"
    exit 1
fi

url_page0="$url"

next='placeholder'
retry=0
max_retry=5
# these are errors that might fix themselves if we wait
retry_errors=(429 500 502 503 504)

while ! [ -z "$next" ]; do
    res=$(curl -s -A random "$url")

    error="$(echo -n "$res" | jq -e '.error')"
    if [ $? -eq 0 ]; then
        # Quit if not in retry errors
        if ! [[ "${ayy}" =~ $error ]]; then
            echo "Error: critical http error $error, exiting"
            exit 1
        fi

        echo "Warning: returned http error $error"

        if [ $retry -eq $max_retry ]; then
            echo "Error: Errored out 5 times in a row, exiting"
            exit 1
        fi

        # wait increasingly longer times
        sleep $((1 + $retry * 10))
        retry=$((retry + 1))

        continue
    fi

    echo "$res" | jq -r '.data.children[].data.url'

    # get next page url, // empty replaces null with empty string
    next=$(echo -n "$res" | jq -r '.data.after // empty')
    url="$url_page0"'&after='"$next"
done