Quick prototype of a variable filtered bitbake -e

README.md

bitbake-env: improved version of bitbake -e

To install

The script may be run from anywhere, as long as 'bitbake' can be found in your PATH.

Suggestion: alias bbe=bitbake-env

Usage

bitbake-env [options] [variable...]

If no variables are specified, all are shown.

Options:
  -h, --help            show this help message and exit
  -r RECIPE, --recipe=RECIPE
                        operate against this recipe
  -f, --flags           also show variable flags
  -d, --dependencies    also show function dependencies

Examples

$ bitbake-env DISTRO MACHINE TUNE_ARCH TUNE_FEATURES
# DISTRO="mel"
# MACHINE="p4080ds"
# TUNE_ARCH="${@bb.utils.contains("TUNE_FEATURES", "m32", "powerpc", "", d)}"
TUNE_ARCH="powerpc"
# TUNE_FEATURES="${TUNE_FEATURES_tune-${DEFAULTTUNE}}"
TUNE_FEATURES="m32 fpu-hard ppce500mc"

$ bitbake-env -d -f COPYLEFT_RECIPE_TYPE
# COPYLEFT_RECIPE_TYPE="${@copyleft_recipe_type(d)}"
COPYLEFT_RECIPE_TYPE="target"
COPYLEFT_RECIPE_TYPE[doc]="The "type" of the current recipe (e.g. target, native, cross)"
def copyleft_recipe_type(d):
    for recipe_type in oe.data.typed_value('COPYLEFT_AVAILABLE_RECIPE_TYPES', d):
        if oe.utils.inherits(d, recipe_type):
            return recipe_type
    return 'target'

$ bitbake-env -r virtual/kernel PROVIDES
Parsing recipes..done.
PROVIDES="linux-qoriq-sdk-3.0.34 linux-qoriq-sdk-3.0.34-r9b linux-qoriq-sdk  virtual/kernel"

Differences compared to bitbake -e

• Filtering: only show the variables you want to see

  • Optionally show function dependencies of the requested variables

• Optionally also show variable flags

• Don't show both commented (unexpanded) and uncommented when the values of the two are one and the same

• Sort sanely: first by type (unexported vars, exported vars, shell functions, python functions), then by dependency order for functions, then by string comparison of variable name (case insensitive)

• Allow showing metadata of recipes in ASSUME_PROVIDED

• Correctly show methodpool functions (bug in bitbake proper, not a feature)

bitbake-env

#!/usr/bin/env python
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

import logging
import optparse
import os
import sys
import warnings

PATH = os.getenv('PATH').split(':')
bitbake_paths = [os.path.join(path, '..', 'lib')
                 for path in PATH if os.path.exists(os.path.join(path, 'bitbake'))]
if not bitbake_paths:
    sys.exit("Unable to locate bitbake, please ensure PATH is set correctly.")

sys.path[0:0] = bitbake_paths

import bb
import bb.cache
import bb.codeparser
import bb.tinfoil


logger = logging.getLogger('bitbake-env')


def get_data(tinfoil, recipe=None):
    localdata = bb.data.createCopy(tinfoil.config_data)
    localdata.finalize()
    # TODO: why isn't expandKeys a method of DataSmart?
    bb.data.expandKeys(localdata)

    if recipe:
        taskdata = bb.taskdata.TaskData(abort=False)
        taskdata.add_provider(localdata, tinfoil.cooker.status, recipe)

        targetid = taskdata.getbuild_id(recipe)
        fnid = taskdata.build_targets[targetid][0]
        fn = taskdata.fn_index[fnid]

        try:
            envdata = bb.cache.Cache.loadDataFull(fn, tinfoil.cooker.get_file_appends(fn),
                                                  tinfoil.config_data)
        except Exception:
            logger.exception("Unable to read %s", fn)
            raise
        return envdata
    return localdata


# TODO: enhance bb.data.emit* to function more flexibly, like these
def escape_shell_value(value):
    value = value.replace('$', '\$')
    value = value.replace('"', '\"')
    value = value.replace('`', '\`')
    return value


def format_variable(data, variable, flag=None, shell=False):
    if flag:
        unexpanded = data.getVarFlag(variable, flag, False)
        pattern = '%s[%s]="%%s"' % (variable, flag)
    else:
        unexpanded = data.getVar(variable, False)
        pattern = '%s="%%s"' % variable

    if data.getVarFlag(variable, 'unexport'):
        if flag:
            return pattern % unexpanded
        else:
            return '# ' + pattern % unexpanded

    try:
        expanded = bb.data.expand(unexpanded, data)
    except BaseException:
        if flag:
            logger.exception("Expansion of '%s[%s]' failed", variable, flag)
        else:
            logger.exception("Expansion of '%s' failed", variable)
        return '# ' + pattern % unexpanded
    else:
        message = ''
        if unexpanded != expanded:
            message += '# ' + pattern % unexpanded + '\n'

        if data.getVarFlag(variable, 'export'):
            message += 'export '
        message += pattern % expanded
        return message


ignored_flags = ('func', 'python', 'export', 'export_func')
def print_variable_flags(data, variable):
    flags = data.getVarFlags(variable)
    if not flags:
        return

    for flag, value in flags.iteritems():
        if flag.startswith('_') or flag in ignored_flags:
            continue
        value = str(value)
        print(format_variable(data, variable, flag))


def print_variable(data, variable):
    unexpanded = data.getVar(variable, False)
    if unexpanded is None:
        return
    unexpanded = str(unexpanded)


    flags = data.getVarFlags(variable) or {}
    if flags.get('func'):
        try:
            value = bb.data.expand(unexpanded, data)
        except BaseException:
            logger.exception("Expansion of '%s' failed", variable)
            return

        if flags.get('python'):
            # TODO: fix bitbake to show methodpool functions sanely like this
            if variable in bb.methodpool._parsed_fns:
                print(value)
            else:
                print("python %s () {\n%s}\n" % (variable, value))
        else:
            print("%s () {\n%s}\n" % (variable, value))
    else:
        print(format_variable(data, variable, shell=True))


class Formatter(bb.msg.BBLogFormatter):
    def __init__(self, fmt=None, datefmt=None, output=sys.stdout):
        bb.msg.BBLogFormatter.__init__(self, fmt, datefmt)
        self.output = output

    def enable_color(self):
        self.color_enabled = True


# TODO: Let bb.tinfoil.Tinfoil support output files other than stdout, and to
# enable color support in the formatter when it's a tty.
class Tinfoil(bb.tinfoil.Tinfoil):
    def __init__(self, output=sys.stdout):
        # Needed to avoid deprecation warnings with python 2.6
        warnings.filterwarnings("ignore", category=DeprecationWarning)

        # Set up logging
        self.logger = logging.getLogger('BitBake')
        console = logging.StreamHandler(output)
        format = Formatter("%(levelname)s: %(message)s", output=output)
        if output.isatty():
            format.enable_color()
        bb.msg.addDefaultlogFilter(console)
        console.setFormatter(format)
        self.logger.addHandler(console)

        initialenv = os.environ.copy()
        bb.utils.clean_environment()
        self.config = bb.tinfoil.TinfoilConfig(parse_only=True)
        self.cooker = bb.cooker.BBCooker(self.config, self.register_idle_function,
                                         initialenv)
        self.config_data = self.cooker.configuration.data
        bb.providers.logger.setLevel(logging.ERROR)
        self.cooker_data = None


def variable_function_deps(data, variable, deps, seen):
    variable_deps = deps and deps.get(variable) or set()
    if data.getVarFlag(variable, 'python'):
        # TODO: Fix generate_dependencies to return python function
        # execs dependencies, which seem to be missing for some reason
        parser = bb.codeparser.PythonParser(variable, logger)
        parser.parse_python(data.getVar(variable, False))
        variable_deps |= parser.execs

    for dep in variable_deps:
        if dep in seen:
            return
        seen.add(dep)

        if data.getVarFlag(dep, 'func'):
            for _dep in variable_function_deps(data, dep, deps, seen):
                yield _dep
            yield dep

def dep_ordered_variables(data, variables, deps):
    seen = set()
    for variable in variables:
        seen.add(variable)
        for dep in variable_function_deps(data, variable, deps, seen):
            yield dep
        yield variable

def sorted_variables(data, variables=None, show_deps=True):
    def key(v):
        # Order: unexported vars, exported vars, shell funcs, python funcs
        if data.getVarFlag(v, 'func'):
            return int(bool(data.getVarFlag(v, 'python'))) + 2
        else:
            return int(bool(data.getVarFlag(v, 'export')))

    all_variables = data.keys()
    if not variables:
        variables = sorted(all_variables, key=lambda v: v.lower())
        variables = filter(lambda v: not v.startswith('_'), variables)
    else:
        for variable in variables:
            if variable not in all_variables:
                logger.warn("Requested variable '%s' does not exist", variable)
        if show_deps:
            deps = bb.data.generate_dependencies(data)[1]
            variables = list(dep_ordered_variables(data, variables, deps))

    variables = sorted(variables, key=key)
    return variables

def main(args):
    parser = optparse.OptionParser(usage="%prog [options] [variable...]\n\n"
                                         "If no variables are specified, all are shown.")
    parser.add_option("-r", "--recipe", help="operate against this recipe")
    parser.add_option("-f", "--flags", help="also show variable flags",
                      action="store_true")
    parser.add_option("-d", "--dependencies", help="also show function dependencies",
                      action="store_true")
    options, variables = parser.parse_args(args)

    log_format = Formatter("%(levelname)s: %(message)s")
    if sys.stderr.isatty():
        log_format.enable_color()
    console = logging.StreamHandler(sys.stderr)
    console.setFormatter(log_format)
    console.setLevel(logging.INFO)
    logger.addHandler(console)

    tinfoil = Tinfoil(output=sys.stderr)
    tinfoil.prepare(config_only=True)

    ignore = tinfoil.config_data.getVar("ASSUME_PROVIDED", True) or ""
    if options.recipe and options.recipe in ignore.split():
        logger.warn("%s is in ASSUME_PROVIDED" % options.recipe)

    # Let us show the recipe even if it is in ASSUME_PROVIDED
    # TODO: let bitbake -e support showing assumed recipes, the way this does
    tinfoil.config_data.setVar("ASSUME_PROVIDED", "")

    if options.recipe:
        tinfoil.parseRecipes()

    if options.recipe:
        data = get_data(tinfoil, options.recipe)
    else:
        data = get_data(tinfoil)

    variables = sorted_variables(data, variables, options.dependencies)
    for variable in variables:
        print_variable(data, variable)
        if options.flags:
            print_variable_flags(data, variable)


if __name__ == '__main__':
    main(sys.argv[1:])

Nice share.