list-precomp.py

#!/usr/bin/env python
#
# Lists best candidates for adding to pre-compiled headers from an
# output list showing all the includes during a build.
# 
# By Noel Llopis - April 2005
# http://www.gamesfromwithin.com
#
# Modified by Cameron Hart 2014
# 
# Gist: 9882479
#

import argparse
import logging
import os.path
import re
import string
import sys



class IncludeInfo:
    def __init__(self, name):
        self.name = name
    
    name = ""
    ocurrences = 1    
    ocurrences_directly = 0
    indent = 100
    includes = 0
    pch = False
    

def clean_path(path):
    return os.path.normcase(os.path.abspath(path))


def clean_paths(paths_in):
    paths_out = []
    if paths_in:
        for path in paths_in:
            paths_out.append(clean_path(path))
    return paths_out


def read_file(filename):
    if not os.path.exists(filename):
        print "File " + filename + " doesn't exist."
        return ()
        
    f = file(filename, "r")
    lines = f.readlines()
    f.close()
    return lines
    
    
def ignore_file(inc, ignore_paths):
    for ignore_path in ignore_paths:
        if string.find(inc, ignore_path) == 0:
            logging.debug('Ignore %s %s' % (inc, ignore_path))
            return 1
    return 0
    
    
def parse_include(indent, inc, ignore_pch, ignore_paths, current_info, include_map):
    if current_info.pch and indent > current_info.indent:
        logging.debug("Pch %d %s" % (indent, inc))
        return current_info

    if inc == ignore_pch:
        info = IncludeInfo(inc)
        info.indent = indent
        info.pch = True
        logging.debug("Pch %d %s" % (indent, inc))
        return info
    
    # Update the number of includes caused by the current include (only the first time we encounter that include though).
    direct_include = (indent <= current_info.indent)

    if not direct_include:
        if current_info.ocurrences_directly == 1:
            current_info.includes += 1
                    
    if ignore_file(inc, ignore_paths):
        logging.debug("Skip %d %s" % (indent, inc))
        info = IncludeInfo(inc)
        #info.indent = current_info.indent
        return info
        
    if include_map.has_key(inc):
        info = include_map[inc]
        info.ocurrences += 1
        if direct_include:
            info.ocurrences_directly += 1
        logging.debug("Add %d %d %s" % (indent, direct_include, inc))
    else:
        info = IncludeInfo(inc)
        info.indent = indent
        if direct_include:
            info.ocurrences_directly += 1        
        include_map[inc]=info
        logging.debug("New %d %d %s" % (indent, direct_include, inc))
        
    if direct_include:
        current_info = info            
        current_info.indent = indent
        logging.debug("OK %d %s" % (indent, inc))
        
    return current_info

    
def parse_includes(lines, ignore_pch, ignore_paths, lines_format):
    
    if lines_format == 'msvc':
        include_regex  = re.compile('(\d+\>\s+)?Note: including file:(\s+)(.*)')
        indent_group = 2
        path_group = 3
    elif lines_format == 'gcc':
        include_regex = re.compile('(\.+) (.*)')
        indent_group = 1
        path_group = 2
    else:
        print 'Unknown format: ' + lines_format
        return {}

    if ignore_pch:
        ignore_pch = clean_path(ignore_pch)
    ignore_paths = clean_paths(ignore_paths)
    include_map = {}
    current_info = IncludeInfo("")
    
    for line in lines:
        m = include_regex.match(line)
        if m:
            indent = len(m.group(indent_group))-1
            inc = clean_path(m.group(path_group))
            current_info = parse_include(indent, inc, ignore_pch, ignore_paths,
                current_info, include_map)
                                    
    return include_map

    
def count_cpp_files(lines):

    cpp_count = 0
    cpp_regexp = re.compile('\w+\.(cpp|c)\s*')
    
    for line in lines:
        m = cpp_regexp.search(line)
        if m:
            cpp_count += 1

    return cpp_count  

    
def print_top_includes( include_map, cpp_file_count, top_file_count ):
    
    print "Cpp files: ", cpp_file_count
    print "(Header file, score, times included, includes caused by the header)"               
    result = []
    for key in include_map.keys():
        if include_map[key].ocurrences_directly > 0:
            score = include_map[key].ocurrences * (include_map[key].includes+1)
            result.append( (key, score, include_map[key].ocurrences,
                include_map[key].includes) )
       
    result.sort( lambda a,b: cmp(a[1], b[1]) )
    result.reverse()
    for i in xrange(min(top_file_count,len(result))):
        print result[i]


def main(argv):
    parser = argparse.ArgumentParser(description=
        'Lists best candidates for adding to pre-compiled headers from an output list showing all the includes during a build')
    parser.add_argument('--verbose', action='store_true',
            help='enable verbose output')
    parser.add_argument('--format', choices=['msvc', 'gcc'], required=True,
            help='compiler used to generate include information')
    parser.add_argument('include_trace_file',
            help='path to the compiler include output')
    parser.add_argument('--ignore-pch', nargs='?',
            help='ignore includes from the given pch file')
    parser.add_argument('--ignore-path', action='append',
            help='ignore includes from given path')
    args = parser.parse_args()

    # import pdb
    # pdb.set_trace()

    if args.verbose:
        logging.basicConfig(format='%(message)s',
            level=logging.DEBUG)

    filename = args.include_trace_file
    print "Counting includes in " + filename
    text = read_file(filename)
    if len(text) == 0:
        return
        
    include_map = parse_includes(text, args.ignore_pch, args.ignore_path,
            args.format)
    cpp_file_count = count_cpp_files(text)
    print_top_includes( include_map, cpp_file_count, 30 )
           

if __name__ == "__main__":
    main( sys.argv )