Sample the private dirty memory usage of a process and its children periodically during its execution

example-bitbake-e.txt

# The 'stats' util is from https://github.com/rustyrussell/stats
$ rm -rf cache* tmp*; ./meminfo bitbake -e autoconf 2>log >/dev/null; grep '^meminfo:' log | stats --skip=1 | sort; grep '^meminfo\ total:' log | stats | sort
meminfo: 21948  KnottyUI 9.800000-63.600000(48.4172+/-11)MiB
meminfo: 21948  ProcessEQueue 6.800000-63.000000(59.485+/-12)MiB
meminfo: 21948  python3 3.900000-17.700000(11.4333+/-5.7)MiB
meminfo: 21977  Cooker 1.500000-291.300000(105.685+/-76)MiB
meminfo: 21977  Cooker  984.0KiB
meminfo: 21987  Cooker 6.500000-48.100000(35.3152+/-11)MiB
meminfo: 21989  Parser-1:2 16.800000-82.000000(69.514+/-13)MiB
meminfo: 21990  Parser-1:3 17.000000-81.400000(68.938+/-13)MiB
meminfo: 21991  Parser-1:4 17.000000-86.800000(70.916+/-15)MiB
meminfo: 21992  Parser-1:5 17.000000-87.200000(71.138+/-15)MiB
meminfo: 21993  Parser-1:6 17.200000-80.400000(67.788+/-13)MiB
meminfo: 21994  Parser-1:7 1.000000-81.700000(67.38+/-16)MiB
meminfo: 21995  Parser-1:8 17.800000-80.800000(67.16+/-13)MiB
meminfo: 21996  Parser-1:9 17.000000-87.800000(71.53+/-15)MiB
meminfo: 21997  Parser-1:10 17.400000-80.700000(67.894+/-13)MiB
meminfo: 21998  Parser-1:11 17.000000-83.700000(69.594+/-14)MiB
meminfo: 21999  Parser-1:12 1.000000-85.400000(70.684+/-17)MiB
meminfo: 22000  Parser-1:13 17.300000-90.900000(72.374+/-16)MiB
meminfo: 22001  Parser-1:14 17.300000-84.200000(69.31+/-14)MiB
meminfo: 22002  Parser-1:15 16.800000-83.400000(69.63+/-14)MiB
meminfo: 22003  Parser-1:16 17.000000-87.800000(68.5885+/-14)MiB
meminfo: 22004  Parser-1:17 17.100000-83.500000(70.228+/-14)MiB
meminfo: 22005  Parser-1:18     1000.0KiB
meminfo: 22005  Parser-1:18 17.000000-80.600000(67.8918+/-13)MiB
meminfo: 22006  Parser-1:19 17.100000-89.600000(71.658+/-17)MiB
meminfo: 22007  Parser-1:20 17.300000-80.600000(68.526+/-13)MiB
meminfo: 22008  Parser-1:21 17.100000-81.300000(68.332+/-13)MiB
meminfo: 22009  Parser-1:22 25.000000-91.400000(77.8125+/-14)MiB
meminfo: 22010  Parser-1:23 24.900000-81.300000(69.8061+/-11)MiB
meminfo: 22011  Parser-1:24 24.900000-91.200000(73.8+/-14)MiB
meminfo: 22012  Parser-1:25 24.900000-82.300000(69.3796+/-11)MiB
meminfo: 22013  Parser-1:26 23.700000-87.400000(72.8755+/-13)MiB
meminfo: 22014  Parser-1:27 24.700000-86.900000(71.249+/-13)MiB
meminfo: 22015  Parser-1:28 24.100000-81.800000(68.3271+/-12)MiB
meminfo: 22016  Parser-1:29 25.700000-91.900000(75.7745+/-15)MiB
meminfo: 22017  Parser-1:30 24.700000-80.200000(69.3449+/-11)MiB
meminfo: 22018  Parser-1:31 25.300000-92.000000(74.451+/-14)MiB
meminfo: 22019  Parser-1:32 24.900000-82.700000(69.7592+/-11)MiB
meminfo: 22020  Parser-1:33 24.900000-82.900000(70.4796+/-11)MiB
meminfo total: 1.400000-2.800000(2.40833+/-0.35)GiB
meminfo total: 3.900000-850.500000(133.515+/-1.3e+02)MiB

example-bitbake-p.txt

# The 'stats' util is from https://github.com/rustyrussell/stats
$ rm -rf cache* tmp*; ./meminfo bitbake -p 2>log >/dev/null; grep '^meminfo:' log | stats --skip=1 | sort; grep '^meminfo\ total:' log | stats | sort
meminfo: 23595  KnottyUI 7.000000-16.800000(14.9456+/-1.4)MiB
meminfo: 23595  ProcessEQueue 17.800000-23.300000(22.975+/-0.99)MiB
meminfo: 23595  python3 3.900000-12.700000(8.3+/-4.4)MiB
meminfo: 23605  Cooker 3.500000-202.100000(81.6015+/-50)MiB
meminfo: 23605  Cooker  956.0KiB
meminfo: 23615  Cooker 3.900000-17.200000(14.4784+/-4.8)MiB
meminfo: 23616  Parser-1:2 1.000000-58.700000(39.0222+/-14)MiB
meminfo: 23616  Parser-1:2      1000.0KiB
meminfo: 23618  Parser-1:3 9.000000-60.100000(40.7482+/-13)MiB
meminfo: 23619  Parser-1:4 9.500000-58.700000(40.2855+/-13)MiB
meminfo: 23620  Parser-1:5 9.100000-57.300000(40.0732+/-13)MiB
meminfo: 23621  Parser-1:6 9.200000-58.300000(40.6+/-13)MiB
meminfo: 23622  Parser-1:7 9.500000-59.400000(40.6873+/-13)MiB
meminfo: 23623  Parser-1:8 9.600000-58.900000(41.0375+/-13)MiB
meminfo: 23624  Parser-1:9 8.500000-56.700000(39.8054+/-12)MiB
meminfo: 23625  Parser-1:10 8.900000-58.300000(39.3+/-13)MiB
meminfo: 23626  Parser-1:11 9.100000-60.200000(41.6386+/-14)MiB
meminfo: 23627  Parser-1:12 9.000000-58.100000(40.8268+/-13)MiB
meminfo: 23628  Parser-1:13 8.900000-57.000000(39.9364+/-12)MiB
meminfo: 23629  Parser-1:14     804.0KiB
meminfo: 23629  Parser-1:14 8.900000-57.400000(40.0873+/-13)MiB
meminfo: 23630  Parser-1:15 9.500000-60.600000(40.8291+/-13)MiB
meminfo: 23631  Parser-1:16 9.200000-59.600000(41.3268+/-13)MiB
meminfo: 23632  Parser-1:17 11.900000-58.400000(41.9145+/-12)MiB
meminfo: 23633  Parser-1:18 11.700000-59.200000(41.3273+/-13)MiB
meminfo: 23634  Parser-1:19 1.000000-58.900000(40.2574+/-13)MiB
meminfo: 23634  Parser-1:19     1012.0KiB
meminfo: 23635  Parser-1:20 12.000000-59.900000(41.5981+/-13)MiB
meminfo: 23636  Parser-1:21 11.900000-58.400000(41.7618+/-12)MiB
meminfo: 23637  Parser-1:22 11.800000-60.900000(41.9473+/-13)MiB
meminfo: 23638  Parser-1:23 11.400000-59.400000(41.3764+/-12)MiB
meminfo: 23639  Parser-1:24 12.100000-60.300000(41.3352+/-12)MiB
meminfo: 23640  Parser-1:25 11.900000-59.300000(41.8455+/-13)MiB
meminfo: 23641  Parser-1:26 11.800000-60.700000(42.363+/-13)MiB
meminfo: 23642  Parser-1:27 1.200000-59.000000(40.7491+/-14)MiB
meminfo: 23643  Parser-1:28 11.800000-61.900000(43.1218+/-13)MiB
meminfo: 23644  Parser-1:29 12.000000-59.700000(41.3093+/-12)MiB
meminfo: 23645  Parser-1:30 11.900000-61.800000(42.9036+/-13)MiB
meminfo: 23646  Parser-1:31 11.900000-61.900000(43.0962+/-14)MiB
meminfo: 23647  Parser-1:32 11.700000-60.300000(42.0434+/-13)MiB
meminfo: 23648  Parser-1:33 11.700000-61.300000(42.5981+/-13)MiB
meminfo total: 1.000000-2.000000(1.52889+/-0.32)GiB
meminfo total: 3.900000-1010.800000(152.106+/-2.8e+02)MiB

meminfo

#!/usr/bin/env python3

import psutil
import subprocess
import sys
import time

# Via http://stackoverflow.com/a/1094933
def sizeof_fmt(num, suffix='B'):
    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
        if abs(num) < 1024.0:
            return "%3.1f%s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f%s%s" % (num, 'Yi', suffix)

def main(*argv):
    """Run the specified command and monitor the uss of it and its children

    :param str argv: Command line arguments to run
    """

    queue = []
    p = psutil.Popen(argv)
    while p.poll() is None:
        mem = 0
        processes = p.children(recursive=True)
        processes.insert(0, p)
        for _p in processes:
            with _p.oneshot():
                try:
                    name = _p.name()
                    uss = _p.memory_full_info().uss
                except psutil.NoSuchProcess:
                    continue

                if uss != 0:
                    mem += uss
                    sys.stderr.write('meminfo: %d\t%s\t%s\n' % (_p.pid, name, sizeof_fmt(uss)))

        if mem != 0:
            sys.stderr.write('meminfo total: %s\n' % sizeof_fmt(mem))
        time.sleep(0.1)
    return p.returncode

if __name__ == '__main__':
    sys.exit(main(*sys.argv[1:]))