improved gbs rip distribution generator script for HCS64

00-README.md

See here for an updated version: https://gitgud.io/zumi-gbs/hcs/gbstools

gbsdist.py

#!/usr/bin/python

import pathlib
import os
import shutil
import random
import json
import argparse
import logging
from contextlib import contextmanager

import jsonschema
import py7zr

log = logging.getLogger(__name__)

# JSONSchema for the json file
dist_schema = {
	"type": "object",
	"required": ["file", "cartridge", "meta", "tracks"],
	"properties": {
		"file": {
			"type": "string",
			"description": "[REQUIRED] The GBS rip file to be given tags.",
		},
		"cartridge": {
			"description": "[REQUIRED] Game cartridge info. See https://gbhwdb.gekkio.fi/cartridges/, particular attention is to be drawn to the Releases column.",
			"anyOf": [
				{ "$ref": "#/definitions/cart_properties_1" },
				{ "$ref": "#/definitions/cart_properties_2" }
			]
		},
		"meta": {
			"type": "object",
			"description": "[REQUIRED] Information on the game itself.",
			"$ref": "#/definitions/meta_properties"
		},
		"tracks": {
			"type": "array",
			"description": "[REQUIRED] A list of track info, ordered by playlist number.",
			"uniqueItems": True,
			"minItems": 1,
			"items": {
				"$ref": "#/definitions/track"
			}
		},
		"fade": {
			"type": "integer",
			"description": "[RECOMMENDED] If defined, this will set the default fade time for every track."
		},
		"loop_count": {
			"type": "integer",
			"description": "If defined, this will set the default looping count for every track."
		}
	},
	"definitions": {
		"track": {
			"type": "object",
			"additionalProperties": False,
			"required": [ "name", "number" ],
			"properties": {
				"name": {
					"type": "string",
					"description": "[REQUIRED] Track name"
				},
				"number": {
					"type": "integer",
					"description": "[REQUIRED] Index of track in the GBS file (0-indexed)"
				},
				"fade": {
					"type": "integer",
					"description": "Number of seconds to fade out the track. Overrides the default fade time."
				},
				"loop_count": {
					"type": "integer",
					"description": "Number of loops to play before fading out the song. Overrides the default fade time."
				},
				"length": {
					"type": "string",
					"description": "[RECOMMENDED] Length of track in (hh:)mm:ss format.",
					"pattern": "^([0-9]+:)+[0-9]+$"
				},
				"loop": {
					"type": "string",
					"description": "[RECOMMENDED] Loop length in (hh:)mm:ss format. If the last character is '-', this value is interpreted as the loop start time. If the value itself is '-', the loop length is the same as the play time (use if the song loops back from the beginning).",
					"pattern": "^([0-9]+:)+[0-9]+-?$|^-$"
				},
				"composer": {
					"type": "array",
					"description": "Composer(s) of this particular track.",
					"$ref": "#/definitions/common_name_list"
				},
				"arranger": {
					"type": "array",
					"description": "Arranger(s) of this particular track.",
					"$ref": "#/definitions/common_name_list"
				},
				"sequencer": {
					"type": "array",
					"description": "Sequencer(s) of this particular track.",
					"$ref": "#/definitions/common_name_list"
				},
				"engineer": {
					"type": "array",
					"description": "Engineer(s) of this particular track.",
					"$ref": "#/definitions/common_name_list"
				},
				"comments": {
					"type": "array",
					"description": "Any remarks to be made ripping this particular track. Each element is its own line.",
					"minItems": 1,
					"items": {
						"type": "string"
					}
				}
			}
		},
		
		"meta_properties": {
			"additionalProperties": False,
			"required": [ "title", "date" ],
			"properties": {
				"title": {
					"type": "string",
					"description": "[REQUIRED] Title of the game."
				},
				"alternate_titles": {
					"type": "array",
					"description": "Alternate titles of the game, if any. This can be its Japanese title, for example.",
					"$ref": "#/definitions/common_name_list"
				},
				"qualifiers": {
					"type": "array",
					"description": "Release qualifiers, if any. e.g. 'Unreleased', 'Prototype', 'Beta', etc.",
					"$ref": "#/definitions/common_name_list"
				},
				"date": {
					"type": "string",
					"description": "[REQUIRED] Release date of the game. Uses the YYYY-MM-DD format. Can be shortened to just the month or year.",
					"pattern": "^[0-9]+(-[0-9]{2,})?(-[0-9]{2,})?$|^\\?+$"
				},
				"artist": {
					"type": "array",
					"description": "[RECOMMENDED] This is usually the copyright holder(s) (company or publisher) of the game.",
					"$ref": "#/definitions/common_name_list"
				},
				"composer": {
					"type": "array",
					"description": "[RECOMMENDED] Composer(s) of the soundtrack.",
					"$ref": "#/definitions/common_name_list"
				},
				"arranger": {
					"type": "array",
					"description": "Arranger(s) of the soundtrack.",
					"$ref": "#/definitions/common_name_list"
				},
				"sequencer": {
					"type": "array",
					"description": "Sequencer(s) of the soundtrack.",
					"$ref": "#/definitions/common_name_list"
				},
				"engineer": {
					"type": "array",
					"description": "Engineer(s) of the soundtrack. This can be the person responsible for the sound driver, for example.",
					"$ref": "#/definitions/common_name_list"
				},
				"ripper": {
					"type": "array",
					"description": "[RECOMMENDED] Ripper(s) of the GBS.",
					"$ref": "#/definitions/common_name_list"
				},
				"tagger": {
					"type": "array",
					"description": "[RECOMMENDED] Tagger(s) of the M3U file.",
					"$ref": "#/definitions/common_name_list"
				},
				"comments": {
					"type": "array",
					"description": "Any remarks to be made ripping this M3U file. Each element is its own line.",
					"minItems": 1,
					"items": {
						"type": "string"
					}
				}
			}
		},
		
		"cart_properties_1": {
			"properties": {
				"model": {
					"type": "string",
					"description": "[REQUIRED] Game Boy model code. Can be either: DMG, CGB. This is the 'DMG' part of 'DMG-ADDE-USA-2'.",
					"pattern": "^(DMG|CGB)$"
				},
				"code": {
					"type": "string",
					"description": "[REQUIRED] Internal game code, is an alphanumeric code between 2 to 4 characters long. This is the 'ADDE' part of 'DMG-ADDE-USA-2'.",
					"pattern": "^[A-Z0-9]{2,4}$"
				},
				"region": {
					"type": "string",
					"description": "Three letter regional code, e.g. USA, JPN, AUS, EUR, ITA, NOE (Nintendo of Europe). This is the 'USA' part of 'DMG-ADDE-USA-2'.",
					"pattern": "^[A-Z]{3}$"
				},
				"revision": {
					"type": "integer",
					"description": "Revision number of the ROM. This is the '2' part of 'DMG-ADDE-USA-2'."
				},
				"prefer": {
					"type": "string",
					"description": "Game Boy model name, where the game is commonly played with. Can be either: GB, GBC, SGB.",
					"pattern": "^(GBC?|SGB)$"
				}
			},
			"required": [ "model", "code" ]
		},
		
		"cart_properties_2": {
			"properties": {
				"model": {
					"type": "string",
					"description": "[REQUIRED] Game Boy model name. Can be either: GB, GBC, SGB.",
					"pattern": "^(GBC?|SGB)$"
				}
			},
			"required": [ "model" ]
		},
		
		"common_name_list": {
			"uniqueItems": True,
			"minItems": 1,
			"items": {
				"type": "string"
			}
		}
	}
}


def determine_dist_name(deserialized):
	"""
	Create the name to be used for the 7z file name and title of rip.
	
	:param deserialized: (dict) Playlist info JSON data
	"""
	cart = deserialized["cartridge"]
	
	# if we have the game code, try to use the GB target instead
	if "code" in cart.keys():
		target_model = cart.get("prefer")
		
		if target_model:
			model = target_model
		else:
			# cartridge model needs to be translated
			table = {
				"DMG": "GB",
				"CGB": "GBC"
			}
			model = table[cart["model"]]
	else:  # use the model name directly
		model = cart["model"]
	
	meta = deserialized["meta"]
	entities = meta.get("artist")
	title = meta["title"]
	
	suffix = ""
	
	# add alternate titles like [Akai] [Doukutsu Monogatari] etc...
	alt_titles = meta.get("alternate_titles")
	if alt_titles:
		title += " [%s]" % ('] ['.join(alt_titles))
	
	# add qualifiers like (Prototype)(Rev.B) etc..
	qualifiers = meta.get("qualifiers")
	if qualifiers:
		suffix += "(%s)" % (')('.join(qualifiers))
	
	name = title
	
	# are there publishers/companies defined?
	if entities:
		return "%s %s(%s)(%s)[%s]" % (
			title,
			suffix,
			meta["date"],
			")(".join(entities),
			model
		)
	
	# if there aren't, just omit it
	return "%s %s(%s)[%s]" % (
		title,
		suffix,
		meta["date"],
		model
	)


def determine_m3u_name(deserialized):
	"""
	Create the name to be used for the main M3U and GBS file.
	
	:param deserialized: (dict) Playlist info JSON data
	"""
	cart = deserialized["cartridge"]
	
	# if we have at least the game code, use the internal cart name
	if "code" in cart.keys():
		name = "%s-" % cart["model"]
		name += cart["code"]
		
		region = cart.get("region")
		if region:
			name += "-%s" % region
		
		revision = cart.get("revision")
		if revision:
			name += "-%s" % revision
		
		return name
	
	else: # simply use the 7z name
		return determine_dist_name(deserialized)


def create_main_m3u(deserialized):
	"""
	Create the main M3U string for the GBS distribution.
	
	:param deserialized: (dict) Playlist info JSON data
	:return: (list) List of lines
	"""
	log.info("Processing main m3u file")
	
	lines = []
	
	gbs_file_name = "%s.gbs" % determine_m3u_name(deserialized)
	
	meta = deserialized["meta"]
	
	# add attribution tags
	for tag in ["title", "artist", "composer", "arranger", "sequencer", "engineer", "date", "ripper", "tagger"]:
		if tag in meta.keys():
			if type(meta[tag]) is str:
				value = meta[tag]
			else:  # tag is list
				value = ", ".join(meta[tag])
			log.info("Found tag '%s' = %s" % (tag, value))
			lines.append("# @%-12s%s" % (tag.upper(), value))
	
	# space it out
	lines.append("")  # blank
	
	# add comments
	if "comments" in meta.keys():
		for comment_line in meta["comments"]:
			lines.append("# %s" % comment_line)
		lines.append("")  # blank
	
	tracks = deserialized["tracks"]
	
	for track in tracks:
		# use the track fade duration and loop count, if not available then use the "global" one
		# otherwise, empty it and use the user's.
		fade = track.get("fade", deserialized.get("fade", ""))
		loop_count = track.get("loop_count", deserialized.get("loop_count", ""))
		
		# write the actual lines
		lines.append(
			"%s::GBS,%d,%s,%s,%s,%s,%s" % (
				gbs_file_name,
				track["number"] + 1,  # foo_gep (GME) requires a 1-indexed main m3u
				track["name"].replace(',', '\,'),
				track.get("length", ""),
				track.get("loop", ""),
				fade,
				loop_count,
			)
		)
	return lines


def create_track_m3us(deserialized):
	"""
	Create the M3U strings for each M3U file.
	
	:param deserialized: (dict) Playlist info JSON data
	:return: (dict) Keys are the file names. Contents are a list of lines.
	"""
	log.info("Processing m3u track strings")
	
	# {"playlist_file.m3u": ["Lines of", "the M3U file"}
	files = {}
	
	gbs_file_name = "%s.gbs" % determine_m3u_name(deserialized)
	
	meta = deserialized["meta"]
	tracks = deserialized["tracks"]
	track_number = 0
	
	for track in tracks:
		lines = []
		track_number += 1
		has_metadata = False
		
		# use the track fade duration and loop count, if not available then use the "global" one
		# otherwise, empty it and use the user's.
		fade = track.get("fade", deserialized.get("fade", ""))
		loop_count = track.get("loop_count", deserialized.get("loop_count", ""))
		
		# use the track metadata, if that fails, use the game's metadata, else output a ?
		composer = track.get("composer", []) or meta.get("composer", []) or ["?"]
		arranger = track.get("arranger", []) or meta.get("arranger", []) or []
		sequencer = track.get("sequencer", []) or meta.get("sequencer", []) or []
		engineer = track.get("engineer", []) or meta.get("engineer", []) or []
		
		log.info("...%02d. %s" % (track_number, track["name"]))
		
		# add individual attribution tags
		for tag in ["composer", "arranger", "sequencer", "engineer"]:
			if tag in track.keys():
				has_metadata = True
				if type(track[tag]) is str:
					value = track[tag]
				else:  # tag is list
					value = ", ".join(track[tag])
				log.info("......Found tag '%s' = %s" % (tag, value))
				lines.append("# @%-12s%s" % (tag.upper(), value))
		
		# space it out
		if has_metadata:
			lines.append("")  # blank
		
		# add comments
		if "comments" in track.keys():
			for comment_line in track["comments"]:
				lines.append("# %s" % comment_line)
			lines.append("")  # blank
		
		# merge all the credits into one
		all_in_one = list(dict.fromkeys(composer + arranger + sequencer + engineer))
		
		log.debug("......Computed credits: %s" % all_in_one)
		
		# write the actual lines
		lines.append(
			"%s::GBS,%d,%s - %s - %s - ©%s %s,%s,%s,%s,%s" % (
				gbs_file_name,
				track["number"],  # individual m3us able to be played with in_nez (NEZplug) so no change here
				track["name"].replace(',', '\,'),
				"\, ".join(all_in_one),
				meta["title"].replace(',', '\,'),
				meta["date"],
				"\, ".join(meta.get("artist",["?"])),
				track.get("length", ""),
				track.get("loop", ""),
				fade,
				loop_count
			)
		)
		
		# add it to the definitions
		files["%02d %s.m3u" % (track_number, track["name"])] = lines
	return files


def check_json(json_fn):
	"""
	Check the validity of a JSON file according to this schema.
	
	:param json_fn: JSON file name
	"""
	log.info("Validating json file %s" % json_fn)
	with open(json_fn) as json_file:
		return jsonschema.validate(
			instance=json.load(json_file),
			schema=dist_schema
		)


@contextmanager
def new_temp_dir():
	# generate a random path
	tmp_path = pathlib.Path(
		".tmp-%d" % int(random.random() * 10000)
	)
	
	try:
		# create it and return its name
		log.debug("Creating directory %s" % tmp_path)
		tmp_path.mkdir()
		yield tmp_path
	
	finally:
		# delete the directory
		log.debug("Removing directory %s" % tmp_path)
		shutil.rmtree(tmp_path)


def create_archive(deserialized, out_name="./"):
	"""
	Create the distribution archive.
	
	:param deserialized: (dict) Playlist info JSON data
	:return: None, will output a 7z file.
	"""
	with new_temp_dir() as tmp_dir:
		dist_gbs_path = tmp_dir / ("%s.gbs" % determine_m3u_name(deserialized))
		dist_m3u_path = tmp_dir / ("%s.m3u" % determine_m3u_name(deserialized))
		
		# copy the GBS file with the determined m3u name
		log.debug("%s -> %s" % (
			deserialized["file"],
			dist_gbs_path
		))
		shutil.copy(
			deserialized["file"],
			dist_gbs_path
		)
		
		# create the m3u file
		main_m3u = create_main_m3u(js)
		
		log.info("Writing main m3u file")
		log.debug("...%s" % dist_m3u_path)
		
		with open(dist_m3u_path, "w", encoding="ISO-8859-1") as dist_m3u:
			dist_m3u.write(
				'\r\n'.join(main_m3u)
			)
		
		# then the individual track m3u's
		track_m3u = create_track_m3us(js)
		
		log.info("Writing track m3u files")
		
		for k,v in track_m3u.items():
			dist_track_m3u_path = tmp_dir / k
			
			log.debug("...%s" % dist_track_m3u_path)
			with open(dist_track_m3u_path, "w", encoding="ISO-8859-1") as dist_track_m3u:
				dist_track_m3u.write(
					'\r\n'.join(v)
				)
		
		# calculate 7z output file name
		if out_name[-1] == os.path.sep:
			out_name += "%s.7z" % determine_m3u_name(deserialized)
		
		# create 7zip file
		with py7zr.SevenZipFile(out_name, 'w') as sevenzip:
			for folder, subfolder, files in os.walk(tmp_dir):
				for file_ in files:
					sevenzip.write(
						"%s/%s" % (tmp_dir, file_),
						file_
					)
		
		log.info("Saved to %s!" % out_name)
			

if __name__ == "__main__":
	logging.basicConfig(
		format="%(levelname)8s: %(message)s",
		level=logging.INFO
	)
		
	ap = argparse.ArgumentParser(
		description="Generates a distributable 7z archive of a GBS soundtrack rip suitable for upload to HCS64 and elsewhere.\n"
		"The 7z contains the GBS itself and several NEZPlug-compatible m3u playlists under the format described here: "
		"https://forums.bannister.org/ubbthreads.php?ubb=showflat&Number=78196#Post78196"
	)

	ap.add_argument(
		'json',
		help="Name of the JSON file to parse."
	)

	ap.add_argument(
		'-o', '--output',
		default='./',
		help="Output file. If this ends in a /, it will determine the file name automatically."
	)
	
	args = ap.parse_args()
	
	try:
		check_json(args.json)
	except Exception as e:
		log.critical("Failed to validate file %s!" % args.json)
		log.critical(str(e))
		exit(1)
	
	with open(args.json, "r") as json_file:
		js = json.load(json_file)
		create_archive(js)

katakis3d.json

{
	"file": "katakis3d.gbs",
	"cartridge": {
		"model": "GBC"
	},
	"meta": {
		"title": "Katakis 3D",
		"qualifiers": ["Unreleased"],
		"artist": ["Similis"],
		"composer": ["Tufan Uysal"],
		"date": "2001",
		"ripper": ["zlago"],
		"tagger": ["Zumi"]
	},
	"fade": 10,
	"tracks": [
		{
			"number": 14,
			"name":   "Katakis (Remix)",
			"length": "3:07",
			"loop":   "0:13-"
		},{
			"number": 15,
			"name":   "Loud 'n Proud",
			"length": "0:43",
			"loop":   "0:09-"
		},{
			"number": 1,
			"name":   "Flight to Hell",
			"length": "1:08",
			"loop":   "-"
		},{
			"number": 2,
			"name":   "The Big Thing",
			"length": "0:39",
			"loop":   "-"
		},{
			"number": 3,
			"name":   "Electrical Motions",
			"length": "1:21",
			"loop":   "0:01-"
		},{
			"number": 4,
			"name":   "Protected Beat",
			"length": "0:19",
			"loop":   "-",
			"fade":   5
		},{
			"number": 5,
			"name":   "Secret Cycles",
			"length": "1:52",
			"loop":   "0:30-"
		},{
			"number": 6,
			"name":   "Radioactive Attack",
			"length": "0:32",
			"loop":   "-"
		},{
			"number": 7,
			"name":   "Enforcer",
			"length": "1:20",
			"loop":   "-"
		},{
			"number": 8,
			"name":   "Someone Wanna Party",
			"length": "0:40",
			"loop":   "0:13-"
		},{
			"number": 9,
			"name":   "Rasit's Spiritual Dreams",
			"length": "1:08",
			"loop":   "0:05-"
		},{
			"number": 10,
			"name":   "Oriental Danger",
			"length": "0:40",
			"loop":   "0:19-"
		},{
			"number": 11,
			"name":   "Boomin' Back Katakis",
			"length": "3:51",
			"loop":   "-"
		},{
			"number": 12,
			"name":   "Master of Universe",
			"length": "1:04",
			"loop":   "0:24-"
		},{
			"number": 13,
			"name":   "The Impregnable",
			"length": "0:51",
			"loop":   "-"
		},{
			"number": 16,
			"name":   "30 Seconds to Go...",
			"length": "0:35",
			"loop":   "-"
		},{
			"number": 17,
			"name":   "Beyond the Stars",
			"length": "4:17",
			"loop":   "0:43-"
		},{
			"number": 0,
			"name":   "Crush Boom Bang",
			"length": "0:06",
			"fade":   1
		}
	]
}