QOM type declaration parsing and conversion

alltypes.tsv

convert-qom-types.py

#!/usr/bin/env python3
import sys
import argparse
import os
import os.path
import re
import logging
from typing import Match, Dict, List, Iterable, Tuple, Set, NamedTuple, Union, Optional

logger = logging.getLogger('convert-qom-types')
DBG = logger.debug
INFO = logger.info
WARN = logger.warning

# regex helpers:
def P(s):
    return f'({s})'

def OR(*args):
    return P('|'.join(args))

def S(*args):
    return ''.join(args)

def M(s, t='*'):
    return P(s)+t

def NAMED(name, s):
    return f'(?P<{name}>{s})'

SP = r'\s*'

TI_FIELDS = [ 'name', 'parent', 'abstract', 'interfaces',
    'instance_size', 'instance_init', 'instance_post_init', 'instance_finalize',
    'class_size', 'class_init', 'class_base_init', 'class_data']
RE_TI_FIELD_NAME = OR(*TI_FIELDS)

RE_COMMENT = r'//[^\n]*$|\s*/\*([^*]|\*[^/])*\*/'
RE_COMMENTS = M(S(RE_COMMENT, SP))
RE_IDENTIFIER = r'[a-zA-Z_][a-zA-Z0-9_]*(?![a-zA-Z0-9])'
RE_STRUCT_TYPE = S(r'struct\s+', RE_IDENTIFIER)
RE_TYPE = OR(RE_IDENTIFIER, RE_STRUCT_TYPE)
RE_STRING = r'"([^"\\]|\\[a-z"])*"'
RE_MACRO_CONCAT = M(S(OR(RE_IDENTIFIER, RE_STRING), SP), '{2,}')
RE_NUMBER = r'[0-9]+|0x[0-9a-fA-F]+'
RE_SIMPLE_VALUE = OR(RE_IDENTIFIER, RE_STRING, RE_NUMBER)
RE_FUN_CALL = S(RE_IDENTIFIER, r'\s*\(\s*', RE_SIMPLE_VALUE, r'\s*\)')
RE_SIZEOF = S(r'sizeof\s*\(\s*', RE_TYPE, r'\s*\)')

RE_ARRAY_ITEM = S(r'{\s*', NAMED('arrayitem', M(RE_SIMPLE_VALUE, '?')), r'\s*}\s*,?')
RE_ARRAY_CAST = S(r'\(\s*', RE_IDENTIFIER, r'\s*\[\s*\]\)')
RE_ARRAY_ITEMS = M(S(RE_ARRAY_ITEM, SP))
RE_ARRAY = S(M(RE_ARRAY_CAST, '?'), r'\s*{\s*',
             NAMED('arrayitems', RE_ARRAY_ITEMS),
             r'}')
RE_ADDRESS = S(r'&\s*', RE_IDENTIFIER)
RE_EXPRESSION = OR(RE_FUN_CALL, RE_SIMPLE_VALUE, RE_ARRAY, RE_SIZEOF, RE_ADDRESS, RE_MACRO_CONCAT)

RE_TI_FIELD_INIT = S(NAMED('comments', RE_COMMENTS),
                     r'\.', NAMED('field', RE_TI_FIELD_NAME), r'\s*=\s*',
                     NAMED('value', RE_EXPRESSION), r'\s*,?')
RE_TI_FIELDS = M(S(RE_TI_FIELD_INIT, SP))
RE_TYPEINFO_START = S(r'^\s*((static|const)\s+)*TypeInfo\s+',
                      NAMED('name', RE_IDENTIFIER), r'\s*=\s*{')
RE_STRICT_TYPEINFO_DEF = S(RE_TYPEINFO_START, SP,
                           NAMED('fields', RE_TI_FIELDS),
                           NAMED('endcomments', RE_COMMENTS),
                           r'};?')

def test_res():
    def fullmatch(regexp, s):
        return re.match(regexp+'$', s, re.MULTILINE)

    assert fullmatch(RE_IDENTIFIER, 'sizeof')
    assert fullmatch(RE_IDENTIFIER, 'X86CPU')
    assert fullmatch(RE_FUN_CALL, 'sizeof(X86CPU)')
    assert fullmatch(RE_IDENTIFIER, 'X86_CPU_TYPE_NAME')
    assert fullmatch(RE_SIMPLE_VALUE, '"base"')
    print(RE_FUN_CALL)
    assert fullmatch(RE_FUN_CALL, 'X86_CPU_TYPE_NAME("base")')
    print(RE_TI_FIELD_INIT)
    assert fullmatch(RE_TI_FIELD_INIT, '.name = X86_CPU_TYPE_NAME("base"),')


    assert fullmatch(RE_MACRO_CONCAT, 'TYPE_ASPEED_GPIO "-ast2600"')
    assert fullmatch(RE_EXPRESSION, 'TYPE_ASPEED_GPIO "-ast2600"')

    print(RE_TYPEINFO_START)
    assert re.search(RE_TYPEINFO_START, r'''
    cc->open = qmp_chardev_open_file;
}

static const TypeInfo char_file_type_info = {
    .name = TYPE_CHARDEV_FILE,
#ifdef _WIN32
    .parent = TYPE_CHARDEV_WIN,
''', re.MULTILINE)
    assert re.search(RE_TYPEINFO_START, r'''
        TypeInfo ti = {
            .name = armsse_variants[i].name,
            .parent = TYPE_ARMSSE,
            .class_init = armsse_class_init,
            .class_data = (void *)&armsse_variants[i],
        };''', re.MULTILINE)

    print(RE_ARRAY_ITEM)
    assert fullmatch(RE_ARRAY_ITEM, '{ TYPE_HOTPLUG_HANDLER },')
    assert fullmatch(RE_ARRAY_ITEM, '{ TYPE_ACPI_DEVICE_IF },')
    assert fullmatch(RE_ARRAY_ITEM, '{ }')
    assert fullmatch(RE_ARRAY_CAST, '(InterfaceInfo[])')
    assert fullmatch(RE_ARRAY, '''(InterfaceInfo[]) {
            { TYPE_HOTPLUG_HANDLER },
            { TYPE_ACPI_DEVICE_IF },
            { }
    }''')
    print(RE_COMMENT)
    assert fullmatch(RE_COMMENT, r'''/* multi-line
                                      * comment
                                      */''')

    print(RE_TI_FIELDS)
    assert fullmatch(RE_TI_FIELDS, r'''/* could be TYPE_SYS_BUS_DEVICE (or LPC etc) */
        .parent = TYPE_DEVICE,''')
    assert fullmatch(RE_TI_FIELDS, r'''.name = TYPE_TPM_CRB,
        /* could be TYPE_SYS_BUS_DEVICE (or LPC etc) */
        .parent = TYPE_DEVICE,
        .instance_size = sizeof(CRBState),
        .class_init  = tpm_crb_class_init,
        .interfaces = (InterfaceInfo[]) {
            { TYPE_TPM_IF },
            { }
        }''')
    assert fullmatch(RE_TI_FIELDS + RE_COMMENTS,
        r'''.name = TYPE_PALM_MISC_GPIO,
            .parent = TYPE_SYS_BUS_DEVICE,
            .instance_size = sizeof(PalmMiscGPIOState),
            .instance_init = palm_misc_gpio_init,
            /*
             * No class init required: device has no internal state so does not
             * need to set up reset or vmstate, and has no realize method.
             */''')

    print(RE_STRICT_TYPEINFO_DEF)
    test_empty = r'''
    static const TypeInfo x86_base_cpu_type_info = {
    };
    '''
    assert re.match(RE_STRICT_TYPEINFO_DEF, test_empty, re.MULTILINE)

    test_simple = r'''
    static const TypeInfo x86_base_cpu_type_info = {
        .name = X86_CPU_TYPE_NAME("base"),
        .parent = TYPE_X86_CPU,
        .class_init = x86_cpu_base_class_init,
    };
    '''
    assert re.search(RE_STRICT_TYPEINFO_DEF, test_simple, re.MULTILINE)

    test_interfaces = r'''
    static const TypeInfo acpi_ged_info = {
        .name          = TYPE_ACPI_GED,
        .parent        = TYPE_SYS_BUS_DEVICE,
        .instance_size = sizeof(AcpiGedState),
        .instance_init  = acpi_ged_initfn,
        .class_init    = acpi_ged_class_init,
        .interfaces = (InterfaceInfo[]) {
            { TYPE_HOTPLUG_HANDLER },
            { TYPE_ACPI_DEVICE_IF },
            { }
        }
    };
    '''
    assert re.search(RE_STRICT_TYPEINFO_DEF, test_interfaces, re.MULTILINE)

    test_comments = r'''
    static const TypeInfo palm_misc_gpio_info = {
        .name = TYPE_PALM_MISC_GPIO,
        .parent = TYPE_SYS_BUS_DEVICE,
        .instance_size = sizeof(PalmMiscGPIOState),
        .instance_init = palm_misc_gpio_init,
        /*
         * No class init required: device has no internal state so does not
         * need to set up reset or vmstate, and has no realize method.
         */
    };
    '''
    assert re.search(RE_STRICT_TYPEINFO_DEF, test_comments, re.MULTILINE)

    test_comments = r'''
    static const TypeInfo tpm_crb_info = {
        .name = TYPE_TPM_CRB,
        /* could be TYPE_SYS_BUS_DEVICE (or LPC etc) */
        .parent = TYPE_DEVICE,
        .instance_size = sizeof(CRBState),
        .class_init  = tpm_crb_class_init,
        .interfaces = (InterfaceInfo[]) {
            { TYPE_TPM_IF },
            { }
        }
    };
    '''
    assert re.search(RE_STRICT_TYPEINFO_DEF, test_comments, re.MULTILINE)


ParsedArray = List[str]
ParsedInitializerValue = Union[str, ParsedArray]
class InitializerValue(NamedTuple):
    raw: str
    parsed: Optional[ParsedInitializerValue]
TypeInfoInitializers = Dict[str, InitializerValue]

class TypeInfoData(NamedTuple):
    filename: str
    variable_name: str
    initializers: TypeInfoInitializers

class ParsedFileData(NamedTuple):
    filename: str
    full_types: List[TypeInfoData]
    partial_types: List[TypeInfoData]
    fixed_data: str

def parse_array(m: Match):
    DBG('parse_array: %r', m.group(0))
    return [m.group('arrayitem') for m in re.finditer(RE_ARRAY_ITEM, m.group('arrayitems'))]

def parse_initializer_value(s) -> InitializerValue:
    parsed: Optional[ParsedInitializerValue] = None
    if m := re.match(RE_ARRAY, s):
        assert m is not None
        parsed = parse_array(m)
    return InitializerValue(s, parsed)

def get_typeinfo_initializers(m : Match) -> TypeInfoInitializers:
    return dict((fm.group('field'), parse_initializer_value(fm.group('value')))
                for fm in re.finditer(RE_TI_FIELD_INIT,
                                      m.group('fields')))
def sorted_field_list(l : Iterable[str]):
    return sorted(l, key=lambda v: TI_FIELDS.index(v))

def fix_all_files(args, filenames: List[str]):
    index : Dict[Tuple, List[TypeInfoData]] = {}
    re_strict = re.compile(RE_STRICT_TYPEINFO_DEF, re.MULTILINE)
    re_partial = re.compile(RE_TYPEINFO_START, re.MULTILINE)

    def fix_file_content(fn: str, s: str) -> ParsedFileData:
        full_types: List[TypeInfoData] = []
        def fix_typeinfo(m: Match) -> str:
            DBG('fix_typeinfo: %r', m)
            name = m.group('name')
            initializers = get_typeinfo_initializers(m)
            fields = sorted_field_list(initializers.keys())
            t = TypeInfoData(fn, name, initializers)
            full_types.append(t)
            index.setdefault(tuple(fields), []).append(t)
            return 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n'*10

        DBG('handling %d chars', len(s))
        fixed = re_strict.sub(fix_typeinfo, s)
        partial_types = [TypeInfoData(fn, m.group('name'), {})
                         for m in re_partial.finditer(s)]
        return ParsedFileData(fn, full_types, partial_types, fixed)

    DBG("filenames: %r", filenames)
    alltypes: List[TypeInfoData] = []
    processed: List[TypeInfoData] = []
    for fn in filenames:
        DBG('opening %s', fn)
        f = open(fn, 'rt')
        c = f.read()
        parsed = fix_file_content(fn, c)
        DBG("%s: found %d full types, %d partial types",
            parsed.filename, len(parsed.full_types), len(parsed.partial_types))
        f.close()

        found_full_types = set(t.variable_name for t in parsed.full_types)
        found_partial_types = set(t.variable_name for t in parsed.partial_types)
        incomplete_types = found_partial_types - found_full_types
        if incomplete_types:
            WARN("%s: partial types not parsed completely: %s", fn, ' '.join(incomplete_types))

        processed.extend(parsed.full_types)
        alltypes.extend(parsed.full_types)
        alltypes.extend(t for t in parsed.partial_types
                          if t.variable_name in incomplete_types)

        if not args.dry_run:
            if args.inplace:
                with open(fn+'.changed', 'wt') as f:
                    f.write(parsed.fixed_data)
                os.rename(fn+'.changed', fn)
            else:
                sys.stdout.write(c)
                sys.stdout.flush()

    INFO("%d TypeInfo structs found, %d processed", len(alltypes), len(processed))

    if args.debug:
        items = sorted(index.items(), key=lambda i: -len(i[1]))
        for k,v in items:
            v = index[k]
            fields = sorted_field_list(k)
            DBG("%d TypeInfo structs setting: %s" % (len(v), ', '.join(fields)))
            for (fn,name, initializers) in v:
                DBG("- %s: %s (%s)" % (fn, name, initializers.get('name')))

    if args.table:
        fields = ['filename', 'variable_name'] + TI_FIELDS
        print('\t'.join(fields))
        for t in alltypes:
            values = [t.filename, t.variable_name] + \
                     [t.initializers.get(f, InitializerValue('', '')).raw
                      for f in TI_FIELDS]
            DBG('values: %r', values)
            assert all('\t' not in v for v in values)
            values = [v.replace('\n', ' ').replace('"', '""') for v in values]
            print('\t'.join(values))

def main():
    p = argparse.ArgumentParser()
    p.add_argument('filename', nargs='+')
    p.add_argument('--inplace', '-i', action='store_true')
    p.add_argument('--dry-run', action='store_true')
    p.add_argument('--debug', '-d', action='store_true')
    p.add_argument('--verbose', '-v', action='store_true')
    p.add_argument('--table', action='store_true',
                   help="Print CSV table of type information")
    args = p.parse_args()

    if args.debug:
        logging.basicConfig(level=logging.DEBUG)
    elif args.verbose:
        logging.basicConfig(level=logging.INFO)
    else:
        logging.basicConfig(level=logging.WARN)

    fix_all_files(args, args.filename)

if __name__ == '__main__':
    main()