kergoth/check-python-deps

## check-python-deps
#!/bin/sh
# Use recipetool to check for missing PACKAGECONFIG/RDEPENDS in our python recipes

set -e

python_recipes () {
    bitbake -s | grep '^python-' | grep -v native | awk '{print $1}'
}

get_value () {
    sed -n 's/[^"]*"\([^"]*\)"/\1/p'
}


tempdir="$(mktemp -d -t "${0##*/}.XXXXXX")" || exit 1
trap 'rm -rf "$tempdir"' EXIT INT TERM
recipe_list="$tempdir/recipes"
oldvalue="$tempdir/old"
newvalue="$tempdir/new"


# Gather list of 'python-*' recipes
# My pulseaudio do_configure is hanging, so exclude pyqt to keep it out
python_recipes | grep -Ev 'pyqt' >"$recipe_list"

# Bootstrap the pkgdata for use by recipetool
echo >&2 "Building python and python-distribute to populate pkgdata"
bitbake python python-distribute

# Attempt to populate pkgdata further to pick up interdependencies
# Fmt is used to limit the number of recipes per bitbake command, to avoid
# line length failure in buildstats.
cat "$recipe_list" | tr '\n' ' ' | sed 's,^ *,,; s, *$,,' | fmt -w 150 | \
    while read recipes; do
        echo >&2 "Building to populate pkgdata: $recipes"
        bitbake -k $recipes || true
    done

echo >&2 "Processing python recipes.."
outfile="${0##*/}.txt"
>"$outfile"
while read recipe; do
    SRC_URI="$(bitbake -e "$recipe" | grep "^SRC_URI=" | get_value | tr ' ' '\n' | head -n 1 | sed 's/ //g')" || break
    bitbake -e "$recipe" | grep "^RDEPENDS_$recipe=" | get_value | tr ' ' '\n' | sort -u >"$oldvalue" || break
    if [ -z "$SRC_URI" ]; then
        echo >&2 "Error: no SRC_URI for $recipe, skipping"
        continue
    fi

    recipefile="$tempdir/$recipe.bb"
    recipetool create -o "$recipefile" "$SRC_URI" || continue
    bitbake -e -b "$recipefile" | grep "^RDEPENDS_$recipe=" | get_value | tr ' ' '\n' | grep -vx "$recipe" | sort -u >"$newvalue"
    new_rdepends="$(comm -13 "$oldvalue" "$newvalue" | tr '\n' ' ' | sed 's, *$,,; s,^ *,,')"
    if [ -n "$new_rdepends" ] || grep -q PACKAGECONFIG "$recipefile"; then
        echo "$recipe:"
        if [ -n "$new_rdepends" ]; then
            printf 'RDEPENDS_%s += "%s"\n' "$recipe" "$new_rdepends"
        fi
        grep '^#*PACKAGECONFIG' "$recipefile" || true
    fi | tee -a "$outfile"
done <"$recipe_list"
printf >&2 'See `%s` for all the bits.\n' "$outfile"

## check-python-deps.txt
python-async:
RDEPENDS_python-async += "python-unittest"
python-autobahn:
RDEPENDS_python-autobahn += "python-compression python-core python-crypt python-distribute python-distutils python-io python-json python-lang python-math python-netclient python-pickle python-re python-shell python-subprocess python-textutils python-twisted-core python-ujson python-zopeinterface python-zope-interface"
#PACKAGECONFIG ?= "serialization asyncio compress accelerate twisted"
PACKAGECONFIG[serialization] = ",,,python-msgpack-python"
PACKAGECONFIG[asyncio] = ",,,python-futures python-trollius"
PACKAGECONFIG[compress] = ",,,python-lz4 python-python-snappy"
PACKAGECONFIG[accelerate] = ",,,python-ujson python-wsaccel"
PACKAGECONFIG[twisted] = ",,,python-twisted"
python-cheetah:
RDEPENDS_python-cheetah += "python-codecs python-core python-crypt python-debugger python-distribute python-distutils python-hotshot python-io python-lang python-logging python-markdown python-math python-netserver python-re python-shell python-stringold python-subprocess python-tests python-textutils python-threading python-unittest python-xml"
python-cloudeebus:
RDEPENDS_python-cloudeebus += "python-core python-lang python-re python-setuptools python-twisted-core python-xml"
python-cmd2:
RDEPENDS_python-cmd2 += "python-doctest python-io python-lang python-netclient python-re python-shell python-subprocess python-textutils python-unittest"
python-cython:
RDEPENDS_python-cython += "python-2to3 python-codecs python-compression python-crypt python-distutils python-doctest python-importlib python-io python-lang python-logging python-lxml python-misc python-multiprocessing python-netclient python-numpy python-pickle python-pprint python-re python-stringold python-tests python-textutils python-threading python-unittest python-xml"
python-dateutil:
RDEPENDS_python-dateutil += "python-compression python-io python-shell python-stringold"
python-dbusmock:
RDEPENDS_python-dbusmock += "python-argparse python-dbus python-importlib python-io python-re python-subprocess python-unittest python-xml"
python-decorator:
RDEPENDS_python-decorator += "python-contextlib python-lang python-re"
python-docutils:
RDEPENDS_python-docutils += "python-codecs python-compression python-distutils python-imaging python-io python-lang python-math python-netclient python-pprint python-re python-stringold python-textutils python-unittest python-xml"
python-epydoc:
RDEPENDS_python-epydoc += "python-codecs python-curses python-doctest python-docutils python-hotshot python-io python-lang python-math python-netclient python-netserver python-pickle python-pprint python-profile python-re python-shell python-stringold python-subprocess python-textutils python-unittest python-xml python-zopeinterface"
python-feedparser:
RDEPENDS_python-feedparser += "python-codecs python-compression python-email python-html python-io python-mime python-netclient python-netserver python-re python-stringold python-xml"
python-futures:
RDEPENDS_python-futures += "python-lang python-logging python-multiprocessing python-pkgutil python-threading"
python-gdata:
RDEPENDS_python-gdata += "python-core python-crypt python-db python-distutils python-doctest python-email python-io python-lang python-m2crypto python-math python-misc python-netclient python-netserver python-pickle python-re python-stringold python-twisted-core python-twisted-protocols python-unixadmin python-xml python-xmlrpc"
python-gevent:
RDEPENDS_python-gevent += "python-ctypes python-io python-lang python-math python-netclient python-pyopenssl python-textutils python-threading"
python-git:
RDEPENDS_python-git += "python-datetime python-logging python-mock python-netclient python-nose python-smmap python-unittest python-unixadmin"
python-gitdb:
RDEPENDS_python-gitdb += "python-crypt python-logging python-math python-nose python-unittest"
python-greenlet:
RDEPENDS_python-greenlet += "python-distutils python-lang python-re python-shell python-threading python-unittest"
python-imaging:
RDEPENDS_python-imaging += "python-image python-io python-math python-netclient python-re"
python-lxml:
RDEPENDS_python-lxml += "python-codecs python-crypt python-curses python-cython python-distribute python-distutils python-doctest python-io python-lang python-logging python-math python-misc python-multiprocessing python-netclient python-re python-shell python-stringold python-subprocess python-unittest"
python-m2crypto:
RDEPENDS_python-m2crypto += "python-doctest python-io python-lang python-misc python-netclient python-netserver python-re python-stringold python-twisted-core python-twisted-protocols python-unixadmin python-xmlrpc python-zopeinterface"
python-mako:
RDEPENDS_python-mako += "python-codecs python-compiler python-compression python-core python-distribute python-distutils python-importlib python-io python-lang python-markupsafe python-math python-misc python-nose python-re python-shell python-stringold python-subprocess python-unittest"
#PACKAGECONFIG ?= "beaker"
PACKAGECONFIG[beaker] = ",,,python-beaker"
python-matplotlib:
RDEPENDS_python-matplotlib += "python-distribute python-io python-pycairo python-pygobject python-re python-shell python-subprocess python-textutils"
python-mccabe:
RDEPENDS_python-mccabe += "python-lang python-misc python-textutils"
python-mock:
RDEPENDS_python-mock += "python-contextlib python-lang python-pickle python-pprint python-re python-unittest"
python-nose:
RDEPENDS_python-nose += "python-codecs python-compiler python-compression python-debugger python-difflib python-distribute python-distutils python-doctest python-docutils python-hotshot python-io python-lang python-logging python-math python-multiprocessing python-netclient python-pickle python-re python-shell python-subprocess python-textutils python-threading python-twisted-core python-unittest python-xml"
python-numpy:
RDEPENDS_python-numpy += "python-compiler python-core python-crypt python-ctypes python-curses python-distribute python-io python-lang python-math python-pyrex python-re python-scons python-stringold python-textutils"
python-pep8:
RDEPENDS_python-pep8 += "python-io python-lang python-re python-shell python-textutils"
python-pexpect:
RDEPENDS_python-pexpect += "python-lang python-re python-stringold python-textutils"
python-pip:
RDEPENDS_python-pip += "python-compression python-core python-crypt python-difflib python-distribute python-distutils python-email python-io python-lang python-logging python-math python-netclient python-netserver python-re python-shell python-stringold python-subprocess python-textutils python-threading python-unixadmin python-xmlrpc"
#PACKAGECONFIG ?= "testing"
PACKAGECONFIG[testing] = ",,,python-mock python-nose python-scripttest python-virtualenv"
python-prettytable:
RDEPENDS_python-prettytable += "python-html python-math python-netserver python-re python-textutils"
python-psutil:
RDEPENDS_python-psutil += "python-io python-lang python-netclient python-re python-shell python-subprocess"
python-pyalsaaudio:
RDEPENDS_python-pyalsaaudio += "python-audio python-unittest"
python-pyflakes:
RDEPENDS_python-pyflakes += "python-doctest python-io python-misc python-shell python-subprocess python-textutils python-unittest"
python-pygobject:
RDEPENDS_python-pygobject += "python-ctypes python-distutils python-io python-lang python-logging python-re python-shell python-stringold"
python-pyopenssl:
RDEPENDS_python-pyopenssl += "python-core python-ctypes python-io python-lang python-re python-shell python-subprocess python-unittest"
python-pyparsing:
RDEPENDS_python-pyparsing += "python-debugger python-lang python-re python-stringold"
python-pyparted:
RDEPENDS_python-pyparted += "python-lang python-re"
python-pyserial:
RDEPENDS_python-pyserial += "python-core"
python-pytz:
RDEPENDS_python-pytz += "python-codecs python-distribute python-doctest python-io python-lang python-math python-pprint python-threading"
python-pyudev:
RDEPENDS_python-pyudev += "python-argparse python-contextlib python-core python-lang python-multiprocessing python-netclient python-pygobject python-threading"
python-simplejson:
RDEPENDS_python-simplejson += "python-doctest python-lang python-textutils python-unittest"
python-smmap:
RDEPENDS_python-smmap += "python-io python-math python-unittest"
python-snakefood:
RDEPENDS_python-snakefood += "python-core"
python-sqlalchemy:
RDEPENDS_python-sqlalchemy += "python-2to3 python-compression python-crypt python-distribute python-distutils python-io python-json python-lang python-logging python-math python-netclient python-nose python-pickle python-profile python-re python-shell python-subprocess python-textutils python-threading python-unittest python-xml"
python-twisted:
RDEPENDS_python-twisted += "python-codecs python-compile python-compression python-contextlib python-crypt python-ctypes python-curses python-datetime python-debugger python-distribute python-distutils python-doctest python-email python-hotshot python-importlib python-io python-lang python-logging python-math python-mime python-misc python-netclient python-netserver python-numbers python-pickle python-pprint python-profile python-pygobject python-pyopenssl python-pyserial python-re python-shell python-stringold python-subprocess python-terminal python-textutils python-threading python-twisted-core python-unittest python-unixadmin python-xml python-xmlrpc"
python-vobject:
RDEPENDS_python-vobject += "python-codecs python-crypt python-distribute python-distutils python-doctest python-io python-lang python-logging python-math python-netclient python-python-dateutil python-re python-shell python-stringold python-textutils python-unittest"
python-webdav:
RDEPENDS_python-webdav += "python-crypt python-datetime python-io python-logging python-mime python-netclient python-re python-stringold python-threading python-xml"
python-zopeinterface:
RDEPENDS_python-zopeinterface += "python-distribute python-doctest python-lang python-pkgutil python-unittest"
	#!/bin/sh
	# Use recipetool to check for missing PACKAGECONFIG/RDEPENDS in our python recipes

	set -e

	python_recipes () {
	bitbake -s \| grep '^python-' \| grep -v native \| awk '{print $1}'
	}

	get_value () {
	sed -n 's/[^"]"\([^"]\)"/\1/p'
	}


	tempdir="$(mktemp -d -t "${0##*/}.XXXXXX")" \|\| exit 1
	trap 'rm -rf "$tempdir"' EXIT INT TERM
	recipe_list="$tempdir/recipes"
	oldvalue="$tempdir/old"
	newvalue="$tempdir/new"


	# Gather list of 'python-*' recipes
	# My pulseaudio do_configure is hanging, so exclude pyqt to keep it out
	python_recipes \| grep -Ev 'pyqt' >"$recipe_list"

	# Bootstrap the pkgdata for use by recipetool
	echo >&2 "Building python and python-distribute to populate pkgdata"
	bitbake python python-distribute

	# Attempt to populate pkgdata further to pick up interdependencies
	# Fmt is used to limit the number of recipes per bitbake command, to avoid
	# line length failure in buildstats.
	cat "$recipe_list" \| tr '\n' ' ' \| sed 's,^ ,,; s, $,,' \| fmt -w 150 \| \
	while read recipes; do
	echo >&2 "Building to populate pkgdata: $recipes"
	bitbake -k $recipes \|\| true
	done

	echo >&2 "Processing python recipes.."
	outfile="${0##*/}.txt"
	>"$outfile"
	while read recipe; do
	SRC_URI="$(bitbake -e "$recipe" \| grep "^SRC_URI=" \| get_value \| tr ' ' '\n' \| head -n 1 \| sed 's/ //g')" \|\| break
	bitbake -e "$recipe" \| grep "^RDEPENDS_$recipe=" \| get_value \| tr ' ' '\n' \| sort -u >"$oldvalue" \|\| break
	if [ -z "$SRC_URI" ]; then
	echo >&2 "Error: no SRC_URI for $recipe, skipping"
	continue
	fi

	recipefile="$tempdir/$recipe.bb"
	recipetool create -o "$recipefile" "$SRC_URI" \|\| continue
	bitbake -e -b "$recipefile" \| grep "^RDEPENDS_$recipe=" \| get_value \| tr ' ' '\n' \| grep -vx "$recipe" \| sort -u >"$newvalue"
	new_rdepends="$(comm -13 "$oldvalue" "$newvalue" \| tr '\n' ' ' \| sed 's, $,,; s,^ ,,')"
	if [ -n "$new_rdepends" ] \|\| grep -q PACKAGECONFIG "$recipefile"; then
	echo "$recipe:"
	if [ -n "$new_rdepends" ]; then
	printf 'RDEPENDS_%s += "%s"\n' "$recipe" "$new_rdepends"
	fi
	grep '^#*PACKAGECONFIG' "$recipefile" \|\| true
	fi \| tee -a "$outfile"
	done <"$recipe_list"
	printf >&2 'See `%s` for all the bits.\n' "$outfile"
	python-async:
	RDEPENDS_python-async += "python-unittest"
	python-autobahn:
	RDEPENDS_python-autobahn += "python-compression python-core python-crypt python-distribute python-distutils python-io python-json python-lang python-math python-netclient python-pickle python-re python-shell python-subprocess python-textutils python-twisted-core python-ujson python-zopeinterface python-zope-interface"
	#PACKAGECONFIG ?= "serialization asyncio compress accelerate twisted"
	PACKAGECONFIG[serialization] = ",,,python-msgpack-python"
	PACKAGECONFIG[asyncio] = ",,,python-futures python-trollius"
	PACKAGECONFIG[compress] = ",,,python-lz4 python-python-snappy"
	PACKAGECONFIG[accelerate] = ",,,python-ujson python-wsaccel"
	PACKAGECONFIG[twisted] = ",,,python-twisted"
	python-cheetah:
	RDEPENDS_python-cheetah += "python-codecs python-core python-crypt python-debugger python-distribute python-distutils python-hotshot python-io python-lang python-logging python-markdown python-math python-netserver python-re python-shell python-stringold python-subprocess python-tests python-textutils python-threading python-unittest python-xml"
	python-cloudeebus:
	RDEPENDS_python-cloudeebus += "python-core python-lang python-re python-setuptools python-twisted-core python-xml"
	python-cmd2:
	RDEPENDS_python-cmd2 += "python-doctest python-io python-lang python-netclient python-re python-shell python-subprocess python-textutils python-unittest"
	python-cython:
	RDEPENDS_python-cython += "python-2to3 python-codecs python-compression python-crypt python-distutils python-doctest python-importlib python-io python-lang python-logging python-lxml python-misc python-multiprocessing python-netclient python-numpy python-pickle python-pprint python-re python-stringold python-tests python-textutils python-threading python-unittest python-xml"
	python-dateutil:
	RDEPENDS_python-dateutil += "python-compression python-io python-shell python-stringold"
	python-dbusmock:
	RDEPENDS_python-dbusmock += "python-argparse python-dbus python-importlib python-io python-re python-subprocess python-unittest python-xml"
	python-decorator:
	RDEPENDS_python-decorator += "python-contextlib python-lang python-re"
	python-docutils:
	RDEPENDS_python-docutils += "python-codecs python-compression python-distutils python-imaging python-io python-lang python-math python-netclient python-pprint python-re python-stringold python-textutils python-unittest python-xml"
	python-epydoc:
	RDEPENDS_python-epydoc += "python-codecs python-curses python-doctest python-docutils python-hotshot python-io python-lang python-math python-netclient python-netserver python-pickle python-pprint python-profile python-re python-shell python-stringold python-subprocess python-textutils python-unittest python-xml python-zopeinterface"
	python-feedparser:
	RDEPENDS_python-feedparser += "python-codecs python-compression python-email python-html python-io python-mime python-netclient python-netserver python-re python-stringold python-xml"
	python-futures:
	RDEPENDS_python-futures += "python-lang python-logging python-multiprocessing python-pkgutil python-threading"
	python-gdata:
	RDEPENDS_python-gdata += "python-core python-crypt python-db python-distutils python-doctest python-email python-io python-lang python-m2crypto python-math python-misc python-netclient python-netserver python-pickle python-re python-stringold python-twisted-core python-twisted-protocols python-unixadmin python-xml python-xmlrpc"
	python-gevent:
	RDEPENDS_python-gevent += "python-ctypes python-io python-lang python-math python-netclient python-pyopenssl python-textutils python-threading"
	python-git:
	RDEPENDS_python-git += "python-datetime python-logging python-mock python-netclient python-nose python-smmap python-unittest python-unixadmin"
	python-gitdb:
	RDEPENDS_python-gitdb += "python-crypt python-logging python-math python-nose python-unittest"
	python-greenlet:
	RDEPENDS_python-greenlet += "python-distutils python-lang python-re python-shell python-threading python-unittest"
	python-imaging:
	RDEPENDS_python-imaging += "python-image python-io python-math python-netclient python-re"
	python-lxml:
	RDEPENDS_python-lxml += "python-codecs python-crypt python-curses python-cython python-distribute python-distutils python-doctest python-io python-lang python-logging python-math python-misc python-multiprocessing python-netclient python-re python-shell python-stringold python-subprocess python-unittest"
	python-m2crypto:
	RDEPENDS_python-m2crypto += "python-doctest python-io python-lang python-misc python-netclient python-netserver python-re python-stringold python-twisted-core python-twisted-protocols python-unixadmin python-xmlrpc python-zopeinterface"
	python-mako:
	RDEPENDS_python-mako += "python-codecs python-compiler python-compression python-core python-distribute python-distutils python-importlib python-io python-lang python-markupsafe python-math python-misc python-nose python-re python-shell python-stringold python-subprocess python-unittest"
	#PACKAGECONFIG ?= "beaker"
	PACKAGECONFIG[beaker] = ",,,python-beaker"
	python-matplotlib:
	RDEPENDS_python-matplotlib += "python-distribute python-io python-pycairo python-pygobject python-re python-shell python-subprocess python-textutils"
	python-mccabe:
	RDEPENDS_python-mccabe += "python-lang python-misc python-textutils"
	python-mock:
	RDEPENDS_python-mock += "python-contextlib python-lang python-pickle python-pprint python-re python-unittest"
	python-nose:
	RDEPENDS_python-nose += "python-codecs python-compiler python-compression python-debugger python-difflib python-distribute python-distutils python-doctest python-docutils python-hotshot python-io python-lang python-logging python-math python-multiprocessing python-netclient python-pickle python-re python-shell python-subprocess python-textutils python-threading python-twisted-core python-unittest python-xml"
	python-numpy:
	RDEPENDS_python-numpy += "python-compiler python-core python-crypt python-ctypes python-curses python-distribute python-io python-lang python-math python-pyrex python-re python-scons python-stringold python-textutils"
	python-pep8:
	RDEPENDS_python-pep8 += "python-io python-lang python-re python-shell python-textutils"
	python-pexpect:
	RDEPENDS_python-pexpect += "python-lang python-re python-stringold python-textutils"
	python-pip:
	RDEPENDS_python-pip += "python-compression python-core python-crypt python-difflib python-distribute python-distutils python-email python-io python-lang python-logging python-math python-netclient python-netserver python-re python-shell python-stringold python-subprocess python-textutils python-threading python-unixadmin python-xmlrpc"
	#PACKAGECONFIG ?= "testing"
	PACKAGECONFIG[testing] = ",,,python-mock python-nose python-scripttest python-virtualenv"
	python-prettytable:
	RDEPENDS_python-prettytable += "python-html python-math python-netserver python-re python-textutils"
	python-psutil:
	RDEPENDS_python-psutil += "python-io python-lang python-netclient python-re python-shell python-subprocess"
	python-pyalsaaudio:
	RDEPENDS_python-pyalsaaudio += "python-audio python-unittest"
	python-pyflakes:
	RDEPENDS_python-pyflakes += "python-doctest python-io python-misc python-shell python-subprocess python-textutils python-unittest"
	python-pygobject:
	RDEPENDS_python-pygobject += "python-ctypes python-distutils python-io python-lang python-logging python-re python-shell python-stringold"
	python-pyopenssl:
	RDEPENDS_python-pyopenssl += "python-core python-ctypes python-io python-lang python-re python-shell python-subprocess python-unittest"
	python-pyparsing:
	RDEPENDS_python-pyparsing += "python-debugger python-lang python-re python-stringold"
	python-pyparted:
	RDEPENDS_python-pyparted += "python-lang python-re"
	python-pyserial:
	RDEPENDS_python-pyserial += "python-core"
	python-pytz:
	RDEPENDS_python-pytz += "python-codecs python-distribute python-doctest python-io python-lang python-math python-pprint python-threading"
	python-pyudev:
	RDEPENDS_python-pyudev += "python-argparse python-contextlib python-core python-lang python-multiprocessing python-netclient python-pygobject python-threading"
	python-simplejson:
	RDEPENDS_python-simplejson += "python-doctest python-lang python-textutils python-unittest"
	python-smmap:
	RDEPENDS_python-smmap += "python-io python-math python-unittest"
	python-snakefood:
	RDEPENDS_python-snakefood += "python-core"
	python-sqlalchemy:
	RDEPENDS_python-sqlalchemy += "python-2to3 python-compression python-crypt python-distribute python-distutils python-io python-json python-lang python-logging python-math python-netclient python-nose python-pickle python-profile python-re python-shell python-subprocess python-textutils python-threading python-unittest python-xml"
	python-twisted:
	RDEPENDS_python-twisted += "python-codecs python-compile python-compression python-contextlib python-crypt python-ctypes python-curses python-datetime python-debugger python-distribute python-distutils python-doctest python-email python-hotshot python-importlib python-io python-lang python-logging python-math python-mime python-misc python-netclient python-netserver python-numbers python-pickle python-pprint python-profile python-pygobject python-pyopenssl python-pyserial python-re python-shell python-stringold python-subprocess python-terminal python-textutils python-threading python-twisted-core python-unittest python-unixadmin python-xml python-xmlrpc"
	python-vobject:
	RDEPENDS_python-vobject += "python-codecs python-crypt python-distribute python-distutils python-doctest python-io python-lang python-logging python-math python-netclient python-python-dateutil python-re python-shell python-stringold python-textutils python-unittest"
	python-webdav:
	RDEPENDS_python-webdav += "python-crypt python-datetime python-io python-logging python-mime python-netclient python-re python-stringold python-threading python-xml"
	python-zopeinterface:
	RDEPENDS_python-zopeinterface += "python-distribute python-doctest python-lang python-pkgutil python-unittest"