eotopo_downloader.py

#!/usr/bin/python
# ExplorOz EOTopo MAP
# Download tiles from https://www.exploroz.com/eotopo
# Get some cookies first and add them below.
# To remove empty files when it finishes use: find . -type d -empty -delete
# To compress the files use: advpng -z4
# To build an .mbtiles flies from the resulting folders, use https://github.com/mapbox/mbutil
import sys
import os
import math
import requests
import random
import threading
import queue
import base64
import os.path

from pathlib import Path

headers = {
'Host': 'ts.exploroz.com',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0',
'Accept': 'image/webp,*/*',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'Referer': 'https://www.exploroz.com/eotopo',
'Origin': 'https://www.exploroz.com',
'Connection': 'keep-alive'
}


cookies = {
'CloudFront-Policy': '<cookie here>', 
'CloudFront-Signature': '<cookie here>', 
'CloudFront-Key-Pair-Id': '<cookie here>',
# Add more cookies if necessary.
}

class ThreadClass(threading.Thread):
	def __init__(self, queue):
		threading.Thread.__init__(self)
		self.queue = queue

	def run(self):
		while True:
			arg = self.queue.get()
			download_url(arg[0], arg[1], arg[2])
			self.queue.task_done()

def download_url(zoom, xtile, ytile):

	url = "https://ts.exploroz.com/2019/%d/%d/%d?mb=1" % (zoom, xtile, ytile)
	dir_path = "/root/Toshiba/maps/EOTopo/%d/%d/" % (zoom, xtile)
	download_path = "/root/Toshiba/maps/EOTopo/%d/%d/%d.png" % (zoom, xtile, ytile)

	if not os.path.exists(dir_path):
		try:
			os.makedirs(dir_path)
		except:
			pass
	
	if not os.path.isfile(download_path):
		res = requests.get(url, headers=headers, cookies=cookies)

		if res.status_code == 200:
			print("downloaded %r" % url)
			fp = open(download_path, 'wb')
			fp.write(res.content)
			fp.close()

		elif res.status_code == 404:
			print('empty %r' % url)
			Path(download_path).touch()

		else:
			return

	else:
		pass

if __name__ == '__main__':

	minzoom = 0
	maxzoom = 13

	ranges = {
		# zoom: [(minx, maxx), (miny, maxy)]
		0: [(0, 2), (0, 2)],
		1: [(0, 4), (0, 4)],
		2: [(0, 8), (0, 8)],
		3: [(5, 7), (4, 5)],
		4: [(13, 14), (8, 10)],
		5: [(26, 29), (16, 20)],
		6: [(52, 59), (33, 40)],
		7: [(104, 118), (67, 81)],
		8: [(207, 237), (134, 162)],
		9: [(416, 472), (268, 324)],
		10: [(832, 950), (538, 651)],
		11: [(1665, 1897), (1074, 1302)],
		12: [(3331, 3801), (2149, 2606)],
		13: [(6662, 7602), (4299, 5213)],
	}

	queue = queue.Queue()

	# Create number process
	for i in range(int(sys.argv[1])):
		t = ThreadClass(queue)
		t.setDaemon(True)
		t.start()



	for zoom in range(minzoom, int(maxzoom)+1, 1):
		xtile, final_xtile = ranges[zoom][0]
		ytile, final_ytile = ranges[zoom][1]

		print("%d:%d-%d/%d-%d" % (zoom, xtile, final_xtile, ytile, final_ytile))

		for x in range(xtile, final_xtile + 1):
			for y in range(ytile, final_ytile + 1):
				queue.put((zoom, x, y))

	queue.join()

	try:
		while not queue.empty():
			time.sleep(1)

	except KeyboardInterrupt:
		sys.stdout.write("\r\n")
		sys.stdout.flush()
		sys.exit()