rotmg asset extractor

readme.md

depends on swftools

use python 2.7 (might work on 2.6 or 2.5 if you install argparse)

usage (--help)

usage: extract.py [-h] [-t {production,testing}] [--test] [--prod]
                  [-v VERSION] [-f FILE] [-d DIR] [--archive]

optional arguments:
  -h, --help            show this help message and exit
  -t {production,testing}
  --test                same as "-t testing"
  --prod                same as "-t production"
  -v VERSION            download specific version (default: latest)
  -f FILE, --file FILE  use local swf
  -d DIR, --dir DIR     output directory (default: current directory)
  --archive             write output to a subdir "type-version". If exists, do
                        nothing (no effect with -f)

-t defaults to testing

output

creates 3 subdirectories with:

• obj: any non-xml data from DEFINEBINARY, right now only 3d models
• sheets: spritesheets
• xml: the xml files as-is

also writes concatenated xml files grouped by root tag (Objects.xml etc)

extract.py

#!/usr/bin/python

import re, os, sys
import tempfile
import shutil
import argparse
from subprocess import *
from urllib2 import urlopen, HTTPError

# for parsing swfdump output
TAGRE = re.compile(r'\[(?P<type>\d\d\d)\]\s+(?P<len>\d+)\s+(?P<tag>\S+) defines id (?P<id>\d+)')
EXPORTRE = re.compile(r'exports (?P<id>\d+) as "kabam.rotmg.assets.Embedded(Assets|Data)_(?P<name>\S+?)(CXML|Embed|_)')

# to identify xml parts
XMLRE = re.compile(r'(?:<\?[^>]*>)?\s*<(?P<tag>\w+)>(?P<cont>.*)</\1>', re.M + re.S)
XMLHEADER = re.compile(r'<\?[^>]*>')

TAGS = {
    'DEFINEBINARY': 'txt',
    'DEFINEBITSLOSSLESS2': 'png',
}

EXTRACTFLAGS = {
    'txt': 'b',
    'png': 'p'
}

HOSTS = {
    'testing': 'rotmgtesting.appspot.com',
    'production': 'realmofthemadgod.appspot.com'
}

tempdir = ''
outdir = ''


def die(reason=''):
    if reason:
        print reason
    if tempdir:
        os.chdir('/')
        shutil.rmtree(tempdir)
    sys.exit(1)


def getversion(target):
    print 'reading version.txt'
    try:
        version = urlopen('http://' + HOSTS[target] + '/version.txt').read()
    except:
        die('failed to read version.txt')
    return version


def download_swf(target, version=''):
    print 'downloading swf'
    if not version:
        version = getversion(target)
    name = 'AssembleeGameClient' + version + '.swf'
    try:
        b = urlopen('http://' + HOSTS[target] + '/' + name).read()
    except HTTPError as e:
        die('HTTP error: code %s, reason "%s"' % (e.code, e.reason))
    except:
        die('swf download failed')

    try:
        with open('temp.swf', 'wb') as f:
            f.write(b)
    except:
        die('failed to write temp.swf')

    return version


ids = {}  # 'png': [id1, id2, ...], ...
names = {}  # id: name


def extract(swfname):
    print 'extracting'
    lines = Popen(['swfdump', swfname], stdout=PIPE).communicate()[0].splitlines()
    lines = [x.strip() for x in lines]

    for t in TAGS:
        ids[TAGS[t]] = []

    for l in lines:
        m = TAGRE.match(l)
        if m:
            m = m.groupdict()
            # parsing "DEFINE"
            if m['tag'] in TAGS:
                ids[TAGS[m['tag']]].append(m['id'])
            continue
        m = EXPORTRE.match(l)
        if not m:
            continue
        m = m.groupdict()
        # parsing "exports"
        names[m['id']] = m['name']

    for type in ids:
        for id in ids[type]:
            if id in names:
                name = names[id]
            else:
                name = id
            # extract the thing
            args = ['swfextract',
                '-' + EXTRACTFLAGS[type], id,
                '-o', name + '.' + type, swfname]
            Popen(args).wait()
            sys.stdout.write('.')
    print


def writefile(path, what):
    with open(path, 'wb') as f:
        f.write(what)


def dotxt():
    print 'dealing with txt',
    rev = {}
    sorted = []
    for id in ids['txt']:
        if id not in names:
            continue
        rev[names[id]] = id
    sorted = rev.keys()
    sorted.sort()

    # saving concatenated xml files based on root tag
    files = {}

    for name in sorted:
        fname = name + '.txt'
        with open(fname, 'rb') as f:
            s = f.read()
        m = re.match(XMLRE, s)
        if m is None:
            # detect empty xml
            if not len(s) or re.match(XMLHEADER, s) is not None:
                continue
            # 3d objects
            writefile(os.path.join(outdir, 'obj', name + '.obj'), s)
        else:
            # xml, write separate file and add to concatenated
            writefile(os.path.join(outdir, 'xml', name + '.xml'), s)
            t = m.group('tag')
            if t not in files:
                files[t] = open(os.path.join(outdir, t + '.xml'), 'wb')
                files[t].write('<' + t + '>')
            # prefix each file's content with its name
            files[t].write('\n\n<!' + '-- ' + name + ' -->\n\n')
            files[t].write(m.group('cont'))

    for t in files:
        files[t].write('\n</' + t + '>')
        files[t].close()
    print


def dopng():
    print 'dealing with png',
    for id in ids['png']:
        if id not in names:
            continue
        name = names[id]
        shutil.copy(name + '.png', os.path.join(outdir, 'sheets'))


if __name__ == '__main__':
    p = argparse.ArgumentParser()
    p.add_argument('-t', dest='target', choices=HOSTS.keys(), default='testing')
    p.add_argument('--test', dest='target', action='store_const', const='testing', help='same as "-t testing"')
    p.add_argument('--prod', dest='target', action='store_const', const='production', help='same as "-t production"')
    p.add_argument('-v', dest='version', help='download specific version (default: latest)')
    p.add_argument('-f', '--file', type=argparse.FileType('rb'), help='use local swf')
    p.add_argument('-d', '--dir', help='output directory (default: current directory)')
    p.add_argument('--archive', action='store_true',
        help='write output to a subdir "type-version". If exists, do nothing (no effect with -f)')

    args = p.parse_args()

    outdir = os.getcwd()
    if args.dir:
        outdir = os.path.abspath(args.dir)
    if not (os.path.exists(outdir) and os.path.isdir(outdir)):
        die('invalid output directory')


    tempdir = tempfile.mkdtemp()
    os.chdir(tempdir)

    if args.file:
        with open('temp.swf', 'wb') as f:
            f.write(args.file.read())
    else:
        if not args.version:
            args.version = getversion(args.target)
        if args.archive:
            outdir = os.path.join(outdir, args.target + '-' + args.version)
            if os.path.exists(outdir):
                die('archive already exists')
            try:
                os.mkdir(outdir)
            except:
                die('failed to create archive dir')
        download_swf(args.target, args.version)

    try:
        for d in ['xml', 'obj', 'sheets']:
            p = os.path.join(outdir, d)
            if not os.path.exists(p):
                os.mkdir(p)
    except:
        die('failed to create output directories')

    extract('temp.swf')

    dotxt()
    dopng()

    os.chdir('/')
    shutil.rmtree(tempdir)