Simple python script to install latest retroarch cores from buildbot

download_buildbot_cores.py

#Simple python script to install latest retroarch cores from buildbot
#To use this script, edit the settings below and from the command line, call python "...path_to_file/download_buildbot_cores.py"

## Script settings
#Buidbot URL for your system
buildbot_url = 'http://buildbot.libretro.com/nightly/apple/osx/x86_64/latest/'

#Temporary directory to download files to, you must have write access to this directory
temp_directory = '/Users/enter_username_here/Downloads/temp_cores'
delete_temp_directory_after_complete = False

#Location to install the cores on your system
core_install_location = '/Applications/RetroArch.app/Contents/Resources/cores/'

#Location to save download history, you must have write access to this directory
download_history_location = '/Users/enter_username_here/Documents/RetroArch/cache'

#If True, then the downloader will overwrite the local cores with what is available on the buildbot, otherwise it will check the previous download cache
force_download = False

#List of cores you want to ignore / not install.  Leave this empty if you want all cores
# dont_install_these_cores = []
dont_install_these_cores = ['2048_libretro.dylib.zip','3dengine_libretro.dylib.zip','stonesoup_libretro.dylib.zip','testgl_libretro.dylib.zip','test_libretro.dylib.zip']

chunk_size = 102400 #100 KB chunks for downloading, you shouldnt have to change this

verbose_printing = True #Print out everything thats happening, otherwise, just tell me if it's downloading and installed
## End Script settings

## Start of script
import os, requests, zipfile, json

#Get list of available cores from the buildbot
s = requests.Session()
print('Getting list of available cores...')
r = s.get(buildbot_url+'.index-extended',verify=False,stream=True,timeout=5)

core_data = dict()
core_data['filename'] = [x.split(' ')[-1].strip() for x in r.text.split('\n') if 'zip' in x and os.path.split(x.split(' ')[-1].strip())[-1] not in dont_install_these_cores]
core_data['commit'] = [x.split(' ')[1].strip() for x in r.text.split('\n') if 'zip' in x and os.path.split(x.split(' ')[-1].strip())[-1] not in dont_install_these_cores]
core_data['date'] = [x.split(' ')[0].strip() for x in r.text.split('\n') if 'zip' in x and os.path.split(x.split(' ')[-1].strip())[-1] not in dont_install_these_cores]

# Create a smaller list for script testing
# core_data['filename'] = core_data['filename'][0:3]
# core_data['commit'] = core_data['commit'][0:3]
# core_data['date'] = core_data['date'][0:3]

#Prep directories, load download history
if not os.path.isdir(temp_directory):
	os.mkdir(temp_directory)

if not os.path.isdir(download_history_location):
	os.mkdir(download_history_location)
	download_history = dict()
	download_history['filename'] = list()
	download_history['commit'] = list()
	download_history['date'] = list()
else:
	if os.path.isfile(os.path.join(download_history_location,'buildbot_download_history.json')):
		try:
			with open(os.path.join(download_history_location,'buildbot_download_history.json'), 'r') as fn:
				download_history = json.load(fn)
		except Exception as history_except:
			if verbose_printing:
				print('Error: Unable to load download history, exception: %(history_except)s' % {'history_except': history_except})
			download_history = dict()
			download_history['core_name'] = list()
			download_history['commit'] = list()
			download_history['date'] = list()
	else:
		download_history = dict()
		download_history['core_name'] = list()
		download_history['commit'] = list()
		download_history['date'] = list()

#Download zipped cores from buildbot
cores_to_install = dict()
cores_to_install['core_name'] = list()
cores_to_install['core_name_no_ext'] = list()
cores_to_install['core_fullpath'] = list()
cores_to_install['commit'] = list()
cores_to_install['date'] = list()
for ii, core_to_dl in enumerate(core_data['filename']):
	current_core = dict()
	current_core['filename'] = os.path.split(core_to_dl)[-1]
	current_core['core_name'] = core_to_dl.replace('.zip','')
	current_core['core_name_no_ext'] = os.path.splitext(current_core['core_name'])[0]
	try:
		current_core['commit'] = [x for ii,x in enumerate(core_data['commit']) if current_core['core_name'] in core_data['filename'][ii]][0]
	except:
		current_core['commit'] = None
	try:
		current_core['date'] = [x for ii,x in enumerate(core_data['date']) if current_core['core_name'] in core_data['filename'][ii]][0]
	except:
		current_core['date'] = None
	
	#Check if core is installed, compare it to version in download history
	if verbose_printing:
		print('Checking file %(core_filename)s, version %(core_version)s, date %(core_date)s' % {'core_filename': current_core['filename'], 'core_version': current_core['commit'], 'core_date': current_core['date']})	
	continue_with_download = False
	if os.path.isfile(os.path.join(core_install_location,current_core['core_name'])):
		if current_core['core_name'] in download_history['core_name']:
			idx = download_history['core_name'].index(current_core['core_name'])
			if current_core['commit'] != download_history['commit'][idx]:
				if verbose_printing:
					print('%(core_filename)s version has been updated in buildbot from %(core_version1)s to %(core_version2)s, continuing with download' % {'core_filename': current_core['core_name_no_ext'], 'core_version1': download_history['commit'][idx], 'core_version2': current_core['commit']})
				continue_with_download = True
			else:
				if not force_download:
					if verbose_printing:
						print('%(core_filename)s latest version is already installed, skipping download' % {'core_filename': current_core['core_name_no_ext']})
				continue_with_download = False
		else:
			if verbose_printing:
				print('%(core_filename)s does not appear in download history, continuing with download' % {'core_filename': current_core['core_name_no_ext']})
			continue_with_download = True
	else:
		if verbose_printing:
			print('%(core_filename)s does not appear to be installed yet, continuing with download' % {'core_filename': current_core['core_name_no_ext']})
		continue_with_download = True

	if force_download:
		continue_with_download = True

	if continue_with_download:
		print('Downloading core %(url)s' % {'url': current_core['filename']})
		download_success = False
		try:
			r = s.get(os.path.join(buildbot_url,core_to_dl),verify=False,stream=True,timeout=5)
			with open(os.path.join(temp_directory,core_to_dl),'wb') as core_file:
				size = 0
				for chunk in r.iter_content(chunk_size):
					core_file.write(chunk)
			download_success = True
		except Exception as web_except:
			download_success = False
			try:
				os.remove(os.path.join(temp_directory,core_to_dl)) #Remove bad file if it was generated
			except:
				pass
			if verbose_printing:
				print('Error: Downloading file %(url)s, exception: %(web_except)s' % {'url': current_core['filename'], 'web_except': web_except})
		if download_success:
			if verbose_printing:
				print('Unzipping file %(url)s' % {'url': current_core['filename']})
			unzip_success = False
			try:
				zip_ref = zipfile.ZipFile(os.path.join(temp_directory,core_to_dl),'r')
				zip_ref.extractall(temp_directory)
				zip_ref.close()
				unzip_success = True
			except Exception as zip_except:
				unzip_success = False
				try:
					os.remove(os.path.join(temp_directory,core_to_dl)) #Remove bad zip file if it was generated
				except:
					pass
				if verbose_printing:
					print('Error:  Unzipping file %(url)s, exception: %(zip_except)s' % {'url': current_core['filename'], 'zip_except': zip_except})
			if unzip_success:
				try:
					os.remove(os.path.join(temp_directory,core_to_dl)) #Remove zip file, we're done with it
				except:
					pass
				cores_to_install['core_name'].append(current_core['core_name'])
				cores_to_install['core_name_no_ext'].append(current_core['core_name_no_ext'])
				cores_to_install['core_fullpath'].append(os.path.join(temp_directory,current_core['core_name']))
				cores_to_install['commit'].append(current_core['commit'])
				cores_to_install['date'].append(current_core['date'])

if os.path.isdir(core_install_location):
	for ii,core_to_install in enumerate(cores_to_install['core_fullpath']):
			copy_success = False
			try:
				os.rename(core_to_install,os.path.join(core_install_location,cores_to_install['core_name'][ii]))
				copy_success = True
			except Exception as move_except:
				copy_success = False
				if verbose_printing:
					print('Error copying file %(url)s, exception: %(move_except)s' % {'url': cores_to_install['core_name'][ii], 'move_except': move_except})
			if copy_success:
				if cores_to_install['core_name'][ii] in download_history['core_name']: #In download history, update it
					idx = download_history['core_name'].index(cores_to_install['core_name'][ii])
					download_history['commit'][idx] = cores_to_install['commit'][ii]
					download_history['date'][idx] = cores_to_install['date'][ii]
				else: #Not in download history, add it
					download_history['core_name'].append(cores_to_install['core_name'][ii])
					download_history['commit'].append(cores_to_install['commit'][ii])
					download_history['date'].append(cores_to_install['date'][ii])
				print('Successfully installed %(installed_core)s' % {'installed_core': cores_to_install['core_name_no_ext'][ii]})
else:
	if verbose_printing:
		print('Error: Core directory %(core_dir)s was not found' % {'core_dir': core_install_location})

#Prep temp directory
if delete_temp_directory_after_complete:
	import shutil
	if os.path.isdir(temp_directory):
		if verbose_printing:
			print('Removing temporary download directory')
		shutil.rmtree(temp_directory, ignore_errors=True)
if verbose_printing:
	print('Saving download history')
with open(os.path.join(download_history_location,'buildbot_download_history.json'), 'w') as fn:
	json.dump(download_history,fn)

print('Done!')
## End of script