Skip to content

Instantly share code, notes, and snippets.

Created August 19, 2023 21:39
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
What would you like to do?
Kosmodular grid image scraper
# Quick and dirty script to save external images used on kosmodulargrid directly into the repository.
# This helps prevent images disappearing from the site as their URLs go bad.
# The outputted JSON does need some manual cleanup - do so at your own risk!
# This is free and unencumbered software released into the public domain.
# Clone the repository into the same directory as this script.
import requests
import json
import urllib
import os
with open("kosmodulargrid/public/data/modules.json") as f:
modules_file =
new_modules_file = ""
modules = json.loads(modules_file)
print(f"There are {len(modules)} modules to download.")
for i, item in enumerate(modules):
link = item["imageLink"]
id = item["id"]
#get file extension from url
path = urllib.parse.urlparse(link).path
ext = os.path.splitext(path)[1]
urllib.request.urlretrieve(item["imageLink"], f"kosmodulargrid/public/data/images/{id}{ext}")
except Exception: # lazy, i know
print(f"Download failed for {i}, entry not changed.")
new_modules_file += str(item)
# update module entry with new url and append to file
new_item = item
new_item["imageLink"] = f"{id}{ext}"
new_modules_file += str(new_item)
print(f"Saved {i} of {len(modules)}")
# some find and replace magic is required to clean up this json file but its nothing too tragic
with open("kosmodulargrid/public/data/modules.json", "w") as f:
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment