Compare the dats of two git revisions of a libretro core (for arcade games) to see what is new

Compare arcade dat versions.py

#!/usr/bin/env python3

import re
import sys
from urllib.request import urlopen
from xml.etree import ElementTree

description_matcher = re.compile(r'(.+?)\s*(\[[^\]]+\])$')
class Game():
	def __init__(self, xml):
		self.xml = xml
		self.basename = xml.attrib['name']
		self.parent = xml.attrib.get('cloneof')
		self.bios = None
		romof = xml.attrib.get('romof')
		if romof != self.parent:
			self.bios = romof
		description = xml.findtext('description')
		match = description_matcher.match(description)
		if match:
			self.description = match[1]
			self.status = match[2][1:-1]
		else:
			self.description = description
			self.status = None

		self.year = xml.findtext('year')
		self.manufacturer = xml.findtext('manufacturer')

def datfile_xml_to_dict(xml):
	#We don't need to care about the header I don't think
	#I guess I could implement this using ElementTree.iterparse on the returned HTTP response but eh
	d = {}
	for game in xml.findall('game'):
		d[game.attrib['name']] = Game(game)
	return d

def get_datfiles(repo, path, old_version, new_version=None):
	if not new_version:
		new_version = 'master'
	url = 'https://raw.githubusercontent.com/{0}/{1}/{2}'.format(repo, old_version, path)
	with urlopen(url) as response:
		old_version = datfile_xml_to_dict(ElementTree.parse(response))
	url = 'https://raw.githubusercontent.com/{0}/{1}/{2}'.format(repo, new_version, path)
	with urlopen(url) as response:
		new_version = datfile_xml_to_dict(ElementTree.parse(response))
	return old_version, new_version

def compare_games(basename, old, new):
	changed = False

	if old.description != new.description:
		changed = True
		print(basename, 'has changed description from', old.description, 'to', new.description)

	if old.status != new.status:
		changed = True
		if not new.status and old.status != 'Bootleg':
			#A game no longer being bootleg is just a minor description change and isn't exciting
			print('Yay!', basename, old.description, 'no longer has status:', old.status)
		elif not old.status:
			print('Potential warning!', basename, old.description, 'now has status:', new.status)
		else:
			print('Maybe yay?', basename, old.description, 'has changed status from', old.status, 'to', new.status)

	if old.parent != new.parent:
		changed = True
		print(basename, old.description, 'has changed parent from', old.parent, 'to', new.parent)
	if old.bios != new.bios:
		changed = True
		print(basename, old.description, 'has changed BIOS from', old.bios, 'to', new.bios)

	if old.year != new.year:
		changed = True
		print(basename, old.description, 'has changed year from', old.year, 'to', new.year)

	if old.manufacturer != new.manufacturer:
		changed = True
		print(basename, old.description, 'has changed manufacturer from', old.manufacturer, 'to', new.manufacturer)

	old_driver = old.xml.find('driver')
	new_driver = new.xml.find('driver')
	if old_driver is not None and new_driver is not None:
		old_driver_status = old_driver.attrib.get('status')
		new_driver_status = new_driver.attrib.get('status')
		if old_driver_status != new_driver_status:
			changed = True
			if old_driver_status == 'preliminary':
				print('Yay!', basename, old.description, 'is now', new_driver_status, 'instead of broken')
			elif new_driver_status == 'preliminary':
				#Maybe it was broken all along though, and the driver was just mistakenly labelled as "good"
				print('Warning!', basename, old.description, 'is now broken instead of', old_driver_status, ', old status from name was', old.status)
			else: #The status attrib in these datfiles seems to only ever be preliminary or good, so this might not happen
				print('Maybe yay?', basename, old.description, 'has changed driver status from', old_driver_status, 'to', new_driver_status)

	#Don't think we really need to compare the <video> element? Don't expect that to have actually changed

	old_roms = {rom.attrib['name']: rom for rom in old.xml.findall('rom')}
	new_roms = {rom.attrib['name']: rom for rom in new.xml.findall('rom')}
	renamed_new_roms = []
	for old_rom_name, old_rom in old_roms.items():
		old_merge = old_rom.attrib.get('merge')

		old_crc = old_rom.attrib.get('crc')
		for new_rom_name, nr in new_roms.items():
			new_crc = nr.attrib.get('crc')
			#We don't need to look at size as if that changes then the crc is changed anyway

			if new_rom_name == old_rom_name and new_crc != old_crc:
				changed = True
				if not old_crc:
					print('WARNING!!', basename, old.description, 'has a new ROM that was previously known but undumped!', new_rom_name, new_crc)
				else:
					print('WARNING!!', basename, old.description, 'has changed', old_rom_name, 'CRC from', old_crc, 'to', new_crc)
				break
			if new_crc == old_crc and new_rom_name != old_rom_name:
				#I'm not sure if this should be a scary warning? Not entirely confident whether or not FBNeo core checks the name of the ROM files and not just CRC

				if old_merge:
					renamed_new_roms.append(new_rom_name)

					if not nr.attrib.get('merge'):
						#Only print it if the ROM is from the clone itself, because we are already going to know if something in the parent or BIOS has changed
						changed = True
						print(basename, old.description, 'has had', old_rom_name, 'previously merged from parent/BIOS, and now that is not while also being renamed to', new_rom_name)	
				else:
					if new_rom_name in old_roms and old_roms[new_rom_name].attrib.get('crc') == old_crc:
						#Don't print stuff that just happens to have the same CRC as some other ROM, that's confusing
						continue
					changed = True	
					print(basename, old.description, 'has had rom', old_rom_name, 'renamed to', new_rom_name)
					renamed_new_roms.append(new_rom_name)

				break
			if new_crc == old_crc and new_rom_name == old_rom_name and old_merge and not nr.attrib.get('merge'):
				changed = True
				print('What the', old_rom_name, 'was previously merged from parent/BIOS, and now is not')
				break

	for new_rom_name, new_rom in new_roms.items():
		if new_rom_name not in old_roms and new_rom_name not in renamed_new_roms:
			changed = True
			print('WARNING!!', basename, old.description, 'has a new ROM that was not there before!', ElementTree.tostring(new_rom))

	return changed

def compare_datfiles(old, new):
	renamed_basenames = []
	for basename, game in old.items():
		new_game = None

		for new_basename, ng in new.items():
			if game.description == ng.description and basename != new_basename:
				print('WARNING!!', game.description, 'has been renamed from', basename, 'to', new_basename)
				new_game = ng
				renamed_basenames.append(new_basename)
				break

		if not new_game:
			if basename not in new:
				print('WARNING!!', basename, game.description, 'is removed from the newer datfile!!')
				print('----')
				continue
		
			new_game = new[basename]
		if compare_games(basename, game, new_game):
			print('----')

	print('------------------------------------------------')
	new_games = []
	new_clones = []
	for basename, game in new.items():
		if basename not in old and basename not in renamed_basenames:
			if game.parent:
				new_clones.append(game)
			else:
				new_games.append(game)

	for game in new_games:
		print('New game:', game.basename, game.description)
		if game.bios:
			print('Uses', game.bios, 'BIOS')
		if game.status:
			print('Status:', game.status)
		print(game.manufacturer, '-', game.year)
		print('----')
	print('---------')
	for clone in new_clones:
		print('New clone:', clone.basename, clone.description)
		clone_parent = new[clone.parent]
		print('Parent:', clone.parent, clone_parent.description)
		if clone_parent.bios:
			print('Uses', clone_parent.bios, 'BIOS')
		if clone.status:
			print('Status:', clone.status)
		print(clone.manufacturer, '-', clone.year)
		print('----')

def main():
	#compare current stable 351ELEC against master
	if len(sys.argv) > 1 and sys.argv[1] == 'mame2003':
		old_mame, new_mame = get_datfiles('libretro/mame2003-plus-libretro', 'metadata/mame2003-plus.xml', '829e8b75bf1454e62f07f5b80c2fac746099f681')
		compare_datfiles(old_mame, new_mame)
	else:
		old_fbneo, new_fbneo = get_datfiles('libretro/FBNeo', 'dats/FinalBurn%20Neo%20(ClrMame%20Pro%20XML%2C%20Arcade%20only).dat', '8a12c7a19546e4e41b210eb359e2a65b712deabc')
		compare_datfiles(old_fbneo, new_fbneo)
	
if __name__ == '__main__':
	main()