du vs os.walk

benchmark_du_vs_python.py

#!/usr/bin/env python
# encoding: utf-8
#
# Copyright © 2014 deanishe@deanishe.net
#
# MIT Licence. See http://opensource.org/licenses/MIT
#
# Created on 2014-04-02
#

"""
"""

from __future__ import print_function, unicode_literals

import os
import subprocess
from time import time


directories = map(os.path.expanduser, [
    ...
    ...
    ...
])


def pydu(rootdir):
    s = time()
    size = 0
    for root, dirnames, filenames in os.walk(rootdir):
        for filename in filenames:
            p = os.path.join(rootdir, filename)
            # if os.path.exists(p):
            if os.path.islink(p):
                continue
            try:
                size += os.path.getsize(p)
            except OSError:
                continue
    print('`python` [{} bytes] : {} : {:0.4f} seconds'.format(size, rootdir,
                                                              time() - s))


def du(rootdir):
    cmd = ['du', '-h', rootdir]
    s = time()
    size = subprocess.check_output(cmd).strip().split('\n')[-1]
    print('`du` [{} bytes] : {} : {:0.4f} seconds'.format(size, rootdir,
                                                          time() - s))

for dirpath in directories:
    # du(dirpath)
    pydu(dirpath)
    du(dirpath)

Thanks for posting this!