ssdv_grabber.py

#
#	SSDV Image Grabber
#
#	Author: Mark Jessop <vk5qi@rfhead.net> ~2022
#
#	Dependencies:
#	  * Python 3
#	  * Python libraries (install using pip):
#	    * requests
#	    * wget
#	
#	Run with:
#	python ssdv_grabber.py CALLSIGN TIME
#	where CALLSIGN is the payload callsign, and
#	TIME is a UTC time before the oldest image, as %Y-%m-%dT%H:%M:%SZ
#
#	e.g.: python3 ssdv_grabber.py SP5LOT 2023-05-27T06:00:00Z
#
#	It will first output some statistics of how much of each image was received, and how much
#	of each image each listener received:
#
# 	Listeners: {'SQ2HCE', 'SP5MS', 'SP5LOT/p', 'SP5LOT', 'SQ3DHO', 'SP5MG', 'SP8NCG'}
# 	Getting per Image data...
# 	id,time,total_packets,image_rxed,SQ2HCE,SP5MS,SP5LOT/p,SP5LOT,SQ3DHO,SP5MG,SP8NCG
# 	228498,2023-05-27T08:56:21Z,1321,54.9,0.0,0.0,54.9,0.0,0.0,0.0,0.0
# 	228499,2023-05-27T08:57:29Z,1362,55.3,0.0,0.0,55.3,0.0,0.0,0.0,0.0
# 	228500,2023-05-27T08:59:10Z,2219,34.6,0.0,0.0,34.6,0.0,0.0,0.0,0.0
#	... many more lines
#
#	Note that the image_rxed and the numbers for each callsign is the percentage of the total
#	image packets that were received.
#
#	Then it outputs the total amount of data uploaded by each receiver:
#
# 	SQ2HCE: 41662 packets (10.17 MB)
# 	SP5MS: 148916 packets (36.36 MB)
# 	SP5LOT/p: 64262 packets (15.69 MB)
# 	SP5LOT: 10560 packets (2.58 MB)
# 	SQ3DHO: 37 packets (0.01 MB)
# 	SP5MG: 131880 packets (32.20 MB)
# 	SP8NCG: 254593 packets (62.16 MB)
#
#	.. then it will start downloading each image.
#

import requests
import argparse
import datetime
import json
import sys
import wget

API_URL = "http://ssdv.habhub.org/api/v0/images"
API_TIMEOUT = 60

def get_image_list(callsign="VK5ARG", from_time = ""):# time_delta=-1):
	''' Get a list of SSDV images from the SSDV server, using callsign X, and from the last N days. '''

	#_timedelta = datetime.timedelta(hours=time_delta*24)
	#_dt_now = datetime.datetime.utcnow()
	#_dt_from = (_dt_now + _timedelta).strftime("%Y-%m-%dT%H:%M:%SZ")

	payload = {
		'callsign': callsign,
		'from': from_time
	}

	r = requests.get(API_URL, payload)
	try:
		return r.json()
	except:
		print(r.text)
		raise IOError()


def get_detailed_image_data(image_id=1):
	payload = {
		'include_packets': 'true'
	}

	r = requests.get(API_URL+"/"+str(image_id), payload)
	return r.json()

if __name__ == "__main__":

	_callsign = sys.argv[1]
	_from_time = sys.argv[2]

	_images = get_image_list(callsign=_callsign, from_time=_from_time)

	listeners = set([])
	image_urls = []
	image_ids = []

	for _img in _images:
		_listeners = _img['received_by']
		_url = _img['image_href']
		_id = _img['id']

		for _call in _listeners:
			listeners.add(_call)

		image_urls.append(_url)
		image_ids.append(_id)

	print("Listeners: %s" % str(listeners))

	heard_packets = {}
	for _listener in listeners:
		heard_packets[_listener] = 0

	print("Getting per Image data...")

	_per_image_info = []

	_header = "id,time,total_packets,image_rxed,"

	_header += ",".join(list(listeners))
	print(_header)

	for _id in image_ids:
		_data = get_detailed_image_data(image_id=_id)

		# example output
		# {'type': 'image', 'id': 207678, 'callsign': 'VK5ARG', 'image_id': 33, 'width': 1488, 
		# 'height': 1120, 'subsampling': '2x2', 'quality': 6, 'packet_type': 1, 'packets_received': 1807, 
		# 'packets_missing': 0, 'last_packet': 1806, 'received_eoi': True, 'created': '2022-05-07T00:32:53Z', 
		# 'updated': '2022-05-07T00:33:23Z', 'received_by': ['VK5QI'], 'image_href': 'http://ssdv.habhub.org/images/2022/05/07/2022-05-07--00-32-53-VK5ARG-32B3E.jpeg?u=1807', 
		# 'data_href': 'http://ssdv.habhub.org/images/2022/05/07/2022-05-07--00-32-53-VK5ARG-32B3E.bin?u=1807', 'packets': [{'packet_id': 0, 'eoi': False, 'received_by': ['VK5QI']}, 
		# {'packet_id': 1, 'eoi': False, 'received_by': ['VK5QI']}, {'packet_id': 2, 'eoi': False, 'received_by': ['VK5QI']}, 
		# {'packet_id': 3, 'eoi': False, 'received_by': ['VK5QI']}, {'packet_id': 4, 'eoi': False, 'received_by': ['VK5QI']}, {'packet_id': 5, 'eoi': False, 'received_by': ['VK5QI']}, {'packet_id': 6, 'eoi': False, 'received_by': ['VK5QI']}, {'packet_id': 7, 'eoi': False, 'received_by': ['VK5QI']}, {'packet_id': 8, 'eoi': False, 'received_by': ['VK5QI']}, {'packet_id': 9, 'eoi': False, 'received_by': ['VK5QI']}, {'packet_id': 10, 'eoi': False, 'received_by': ['VK5QI']}, {'packet_id': 11, 'eoi': False, 'received_by': ['VK5QI']}, {'packet_id': 12, 'eoi': False, 'received_by': ['VK5QI']},

		_img_data = {
			'total_image_packets': _data['packets_received'] + _data['packets_missing'],
			'image_received': 100*(_data['packets_received']/ (_data['packets_received'] + _data['packets_missing'])),
			'id': _data['id'],
			'time': _data['created'],
			'listener_packets': {},
			'listener_ratio': {}
		}

		for _call in listeners:
			_img_data['listener_packets'][_call] = 0

		for _packet in _data['packets']:
			for _call in _packet['received_by']:
				heard_packets[_call] += 1
				_img_data['listener_packets'][_call] += 1

		
		_line = f"{_img_data['id']},{_img_data['time']},{_img_data['total_image_packets']},{_img_data['image_received']:.1f}"

		for _call in listeners:
			_img_data['listener_ratio'][_call] = (_img_data['listener_packets'][_call] / _img_data['total_image_packets'])*100.0
			_line += f",{_img_data['listener_ratio'][_call]:.1f}"

		print(_line)

		
	print("")
	print("")
	for _call in heard_packets.keys():
		print("%s: %d packets (%.2f MB)" % (_call, heard_packets[_call], (heard_packets[_call]*256)/1024.0/1024.0))


	#print(get_detailed_image_data(image_id=image_ids[-1]))

	print("Downloading Images")

	for _img in image_urls:
		print("Downloading: %s" % _img)
		wget.download(_img)