Prototype bitbake recipe parser using the python 'lepl' module

README.rst

Prototype bitbake recipe parser using the python 'lepl' module

TODO

• Handle line continuations with \
• Handle \n, \r, etc within values without breaking line continuations
• Handle python functions defined with 'def'. Note: might want to use the line-aware parsing for this.
• Experiment with using lepl's lexer via its Token objects. Note: this might not be possible. To really utilize a lexer with bitbake's file format, it has to be context aware, to avoid parsing the inside of bitbake functions, and I don't know if lepl can handle that.

lepltest.py

#!/usr/bin/env python
#  vim: set fdl=0 et fenc=utf-8 sts=4 sw=4 ts=4 :

from pprint import pprint
from functools import wraps
from collections import namedtuple
from lepl import *
import sys
import logging

def unpack_arg(f):
    @wraps(f)
    def wrapper(arg):
        return f(*arg)
    return wrapper

def unquote(value):
    s = value[0]
    for quote in "'\"":
        if s.startswith(quote) and s.endswith(quote):
            return s[1:-1]
    return s

def any_of(iterable):
    return Or(*[Literal(item) for item in iterable])

Assignment = namedtuple('Assignment', 'identifier operator value')
Assignment = unpack_arg(Assignment)

FlagAssignment = namedtuple('FlagAssignment', 'identifier flag operator value')
FlagAssignment = unpack_arg(FlagAssignment)

Statement = namedtuple('Statement', 'command value')
Statement = unpack_arg(Statement)

Function = namedtuple('Function', 'name body')

@unpack_arg
def new_function(functype, name, body):
    func = Function(name, body)
    if functype == 'python':
        setflag = FlagAssignment([name, "python", "=", "1"])
        return [func, setflag]
    else:
        return func

def export(arg):
    return [FlagAssignment([arg[0], "export", "=", "1"])]

def exported(arg):
    if len(arg) == 4:
        return [Assignment(arg[1:]), export(arg[1])]
    else:
        return Assignment(arg)


newline = ~Literal('\n')
# Note, can't just use Empty for blank, as that one is Drop()ed
blank = Literal('') > (lambda _: None)
spaces = ~Space()[:]
anything = Any()[::'b', ...]

identifier = Regexp(r'[a-zA-Z0-9_/${}]+')
modifier = Any(u'+.:?')
equal = Literal(u'=')
operator = (Literal(u'??=') | Literal(u'?=') | Literal(u':=') |
            (modifier & equal) | (equal & modifier) | equal)
operator = operator > ''.join
value = Regexp(r'[^\n]+') > unquote

flagidentifier = identifier & ~Literal(u'[') & identifier & ~Literal(u']')
path = Regexp(r'[a-zA_Z0-9/+.\-]+')
with DroppedSpace():
    assignment = ~Optional(Literal("export")) & identifier & operator & value
    assignment = assignment > exported

    flagassign = flagidentifier & operator & value
    flagassign = flagassign > FlagAssignment

    statement = any_of([u'inherit', u'include', u'require']) & path
    statement |= Literal(u'addtask') & value
    statement |= Literal(u'addhandler') & value
    statement |= Literal(u'EXPORT_FUNCTIONS') & (identifier[:] > list)
    statement = statement > Statement

    statement |= (~Literal(u'export') & identifier) > export
    statement |= assignment
    statement |= flagassign

    functiontype = Literal(u'python') | blank
    functionstart = (functiontype & identifier & ~Literal(u'()') &
                     ~Literal(u'{') & newline)

    comment = ~Literal("#") & Regexp(r'[^\n]*')

line = (statement | comment | Empty()) & newline

functionend = newline & ~Literal(u'}') & newline
function = functionstart & anything & functionend
function = function > new_function

recipe = (function | line)[:]


if __name__ == '__main__':
    logging.basicConfig()

    functionstart.parse('foo () {\n')
    functionstart.parse('python meh() {\n')
    functionstart.parse('bar_baz () {\n')
    operator.parse('=')
    operator.parse('+=')
    operator.parse('.=')
    operator.parse('=.')
    operator.parse('=+')
    value.parse('"foo"')
    value.parse('"foo bar baz"')
    value.parse("'foo baz'")
    value.parse('test')
    value.parse('"foo bar baz" ')
    assignment.parse('FOO = "bar"')
    assignment.parse('FOO += "bar"')
    assignment.parse('FOO =. "bar"')
    assignment.parse('FOO =+ baz')
    assignment.parse("FOO .= 'bar'")
    assignment.parse("FOO .= 'bar baz'")
    assignment.parse("FOO .= 'bar baz'c")
    assignment.parse("FOO := 'bar'")
    assignment.parse("FOO ?= 'bar'")
    assignment.parse("FOO_${TEST} ??= 'bar'")
    statement.parse('inherit foo')
    statement.parse('require bar')
    statement.parse('export baz')
    statement.parse('include test.inc')
    statement.parse('addtask do_test')
    statement.parse('addtask foo bar')

    function.parse("""foo () {
    bar
    baz
}
""")
    function.parse("""python bar () {
    baz
}
""")
    recipe.parse("""
PATH =. "foo/bin:"
export PATH

testfunc () {
    echo foo
    for i in a b c; do
        echo $i
    done
}
FOO=bar

require conf/foo.conf
inherit autotools
include test

EXPORT_FUNCTIONS foo bar_baz

die() {
	bbfatal "$*"
}

bbnote() {
	echo "NOTE:" "$*"
}

""")

    for filename in sys.argv[1:]:
        try:
            recipefile = open(filename, "rU")
        except (IOError, OSError) as exc:
            sys.exit(str(exc))

        with recipefile:
            pprint(recipe.parse_file(recipefile))