Python script that generates ctags for files and dependencies of a clang compilation json database

ctags-from-clangdb.py

#!/usr/bin/env python
#
# Copyright 2014 Cameron Hart <cam@bitshifter.net.nz>. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#    1. Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#
#    2. Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
# EVENT SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# GistID: 8862263
#
# TODO: Add arguments for ctags exe and ctags arguments
#
import argparse
import json
import logging
import os
import shlex
import subprocess

def parse_args():
    parser = argparse.ArgumentParser(description='generate ctags from Clang Compilcation Database JSON file')
    parser.add_argument('--verbose', action='store_true', help='verbose output')
    parser.add_argument('dbpath', nargs=1, help='path of the Clang Compilation Database JSON file')
    args = parser.parse_args()
    return args

def parse_json(dbpath):
    logging.info('Loading %s', dbpath)
    fd = open(dbpath)
    return json.load(fd)

def parse_command(command):
    argv = shlex.split(command)
    prog, argv = argv[0], argv[1:]
    opts = []
    for opt in argv:
        arg = opt[:2]
        # for now just want includes and defines
        if arg == '-I' or arg == '-D':
            opts.append(opt)
    return (prog, opts)

def process_database(db):
    logging.info('Generating dependencies:')

    files = set()
    for entry in db:
        dir = entry['directory']
        file = entry['file']
        logging.info('%s', os.path.normpath(os.path.join(dir, file)))
        prog, args = parse_command( entry['command'] )
        files |= process_dependencies(prog, dir, file, args)
    return files

def process_dependencies(prog, dir, file, opts):
    results = set()
    args = [ prog, '-M' ] + opts + [ file ]
    devnull = open(os.devnull, 'wb')
    p = subprocess.Popen(args, cwd = dir, stdout = subprocess.PIPE, stderr = devnull)
    output = p.communicate()[0]
    lines = output.split()
    # skip .o
    if len(lines) > 0 and lines[0][-3:] == '.o:':
        lines = lines[1:]
    for line in lines:
        # skip backslashes
        if line == '\\':
            continue
        # handle absolute paths
        if line[0] == os.sep:
            line = os.path.normpath(line)
        else:
            line = os.path.normpath(os.path.join(dir, line))
        results.add(line)
    return results

def generate_tags(files):
    args = shlex.split('ctags --c++-kinds=+p --fields=+liaS --extra=+q -L -')
    logging.info(' '.join(args))
    logging.info("running ctags on %d files" % len(files))
    p = subprocess.Popen(args, stdin = subprocess.PIPE)
    p.communicate(input='\n'.join(files))

def main():
    args = parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.INFO)

    db = parse_json(args.dbpath[0])

    files = process_database(db)

    generate_tags(files)

if __name__ == "__main__":
    main()