Warhammer Underworlds Library card image scrapper

whu-spider.py

import json, urllib.request, urllib.parse

# API url can be found in WHU Library JS source:
# https://warhammerunderworlds.com/wp-content/themes/gw-shadespire/library/js/main.js?ver=5.3
CARDS_JSON_URL = 'https://warhammerunderworlds.com/wp-json/wp/v2/cards/?per_page=300&lang=ENG'

opener = urllib.request.build_opener()
# urllib is banned on the server, so provide an innocent user agent
opener.addheaders = [('User-Agent', 'Mozilla/5.0 (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11')]

response = opener.open(CARDS_JSON_URL)
# decode the response for correct parsing
cards = json.loads(response.read().decode('utf-8'))

for card in cards:
    print('Loading \"{}\" ...'.format(card['title']['rendered']), end='')
    # some of the card names contain unicode characters which have to be escaped
    image_url = urllib.parse.quote(card['acf']['card_image']['url'], safe="/:")
    local_filename = card['acf']['card_image']['filename']
    with opener.open(image_url) as response, open(local_filename, 'wb') as local_file:
        # save in current directory
        local_file.write(response.read())
    print(' saved as \"{}\"'.format(local_filename))