hema_downloader.py

#!/usr/bin/python
# HEMA ExplorerMAP
# Download tiles from http://skippy.hema-labs.com/AUS/ExplorerMap_v1_2/
# To remove empty files when it finishes use: find . -type d -empty -delete
# To compress the files use: advpng -z4
# To build an .mbtiles flies from the resulting folders, use https://github.com/mapbox/mbutil
import sys
import os
import math
import requests
import random
import threading
import queue
import base64
import os.path

from pathlib import Path

headers = {
'Host': 'skippy.hema-labs.com',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0',
'Accept': 'image/webp,*/*',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'Referer': 'skippy.hema-labs.com',
'Origin': 'skippy.hema-labs.com',
}

class ThreadClass(threading.Thread):
	def __init__(self, queue):
		threading.Thread.__init__(self)
		self.queue = queue

	def run(self):
		while True:
			arg = self.queue.get()
			download_url(arg[0], arg[1], arg[2])
			self.queue.task_done()

def download_url(zoom, xtile, ytile):
	
	url = "http://skippy.hema-labs.com/AUS/ExplorerMap_v1_2/%d/%d/%d.png" % (zoom, ytile, xtile)
	dir_path = "/root/Toshiba/maps/ExplorerMap_v1_2/%d/%d/" % (zoom, ytile)
	download_path = "/root/Toshiba/maps/ExplorerMap_v1_2/%d/%d/%d.png" % (zoom, ytile, xtile)

	if not os.path.exists(dir_path):
		try:
			os.makedirs(dir_path)
		except:
			pass
	
	if not os.path.isfile(download_path):
		res = requests.get(url, headers=headers)

		if res.status_code == 200:
			print("downloaded %r" % url)
			fp = open(download_path, 'wb')
			fp.write(res.content)
			fp.close()

		elif res.status_code == 404:
			print('empty %r' % url)
			Path(download_path).touch()

		else:
			print(res.status_code, url)

	#else:
	#	print("skipped %r" % url)

if __name__ == '__main__':

	#lat, lon = -10.68, 109.9
	#final_lat, final_lon = -45, 155.5

	minzoom = 5
	maxzoom = 13

	ranges = {
		# zoom: [(minx, maxx), (miny, maxy)]
		5: [(14, 22), (23, 31)],
		6: [(29, 44), (47, 63)],
		7: [(58, 88), (94, 127)],
		8: [(116, 177), (188, 255)],
		9: [(232, 355), (376, 511)],
		10: [(464, 711), (753, 1023)],
		11: [(1072, 1315), (1656, 1903)],
		12: [(2166, 2617), (3320, 3807)],
		13: [(4330, 5214), (6645, 7611)],
	}

	queue = queue.Queue()

	# Create number process
	for i in range(int(sys.argv[1])):
		t = ThreadClass(queue)
		t.setDaemon(True)
		t.start()

	valid = set()


	for zoom in range(minzoom, int(maxzoom)+1, 1):
		xtile, final_xtile = ranges[zoom][0]
		ytile, final_ytile = ranges[zoom][1]

		print("%d:%d-%d/%d-%d" % (zoom, xtile, final_xtile, ytile, final_ytile))

		for x in range(xtile, final_xtile + 1):
			for y in range(ytile, final_ytile + 1):
				queue.put((zoom, x, y))

	queue.join()

	try:
		while not queue.empty():
			time.sleep(1)

	except KeyboardInterrupt:
		sys.stdout.write("\r\n")
		sys.stdout.flush()
		sys.exit()