pyflymake.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
This is a script which runs and parses the output of various Python code
checking and unit test runner programs to work with flymake. It has lots of
issues, one being that flymake does not seem to show more than one error
message per line of code, meaning that an error or warning which is
intentionally left unfixed can mask an error or warning that would get more
attention.

Additionally, the scripts which check python code are either rather anemic, and
don’t notice too much (pychecker) or are aggressive, and warn about all sorts
of things that they should not (pylint). pep8.py tends to be annoyingly
aggressive about whitespace.

You must have pep8.py, pychecker and pylint in PATH for this script to find
them. Additionally this script attempts to support virtual environments, but
this is largely untested.

For unit tests to be run and failures highlighted, you currently have to use
nose and the nose_machineout plugin (fork of akaihola) version 0.3 from
http://bitbucket.org/akaihola/nose_machineout/

This script is based on the original script from:
 * http://www.emacswiki.org/emacs/PythonMode#toc7
 * http://www.emacswiki.org/emacs-en/PythonProgrammingInEmacs#toc5
 * http://python.pastebin.com/f627691e0
 * http://pastebin.ca/1797770
 * http://paste.uni.cc/20534

Installation
============

Install or make sure you have installed:
 * emacs
 * flymake.el
 * the following python packages:
   * pep8
   * pychecker
   * pylint
   * nose
   * nose_machineout (from http://bitbucket.org/akaihola/nose_machineout/)

Make sure that ``pyflymake.py`` is executable and that ``pep8``,
``pylint`` and ``pychecker`` are in your $PATH.

Add to your .emacs::

    (when (load "flymake" t)
      (defun flymake-pylint-init ()
        (let* ((temp-file (flymake-init-create-temp-buffer-copy
                           'flymake-create-temp-inplace))
               (local-file (file-relative-name
                            temp-file
                            (file-name-directory buffer-file-name))))
          (list "/path/to/pyflymake.py" (list local-file))))

      (add-to-list 'flymake-allowed-file-name-masks
                   '("\\.py\\'" flymake-pylint-init)))

    (global-set-key [f5] 'flymake-display-err-menu-for-current-line)
    (global-set-key [f6] 'flymake-goto-next-error)
    (global-set-key [S-f6] 'flymake-goto-prev-error)

In the root of a source tree in which you want pyflymake to run tests,
create the file ``.pyflymakerc`` with the following content::

    #VIRTUALENV = '/home/me/.virtualenvs/thevirtualenv' # optional
    TEST_RUNNER_COMMAND = 'nosetests'
    TEST_RUNNER_FLAGS = [
        '--verbosity=0',
        '--with-machineout',
        '--machine-output']
    TEST_RUNNER_OUTPUT = 'stderr'

You can use different test runners, too, provided that their output is
similar to nose_machineout's.  For example, Django's test runner could
be used if django-nose is installed::

    TEST_RUNNER_COMMAND = '/home/me/project/manage.py'
    TEST_RUNNER_FLAGS = [
        'test',
        '--settings=test_settings',
        '--failfast',
        '--verbosity=0',
        '--with-machineout',
        '--machine-output']
    TEST_RUNNER_OUTPUT = 'stderr'
"""

import os
import re
import sys
import imp
import logging

from subprocess import Popen, PIPE


class LintRunner(object):
    """ Base class provides common functionality to run
          python code checkers. """

    sane_default_ignore_codes = set([])
    command = None
    output_matcher = None

    #flymake: ("\\(.*\\) at \\([^ \n]+\\) line \\([0-9]+\\)[,.\n]" 2 3 nil 1)
    #or in non-retardate: r'(.*) at ([^ \n]) line ([0-9])[,.\n]'
    output_format = "%(level)s %(error_type)s%(error_number)s:" \
                    "%(description)s at %(filename)s line %(line_number)s."

    def __init__(self, virtualenv=None, ignore_codes=(),
                 use_sane_defaults=True, stream='stdout'):
        if virtualenv:
            # This is the least we can get away with (hopefully).
            self.env = {'VIRTUAL_ENV': virtualenv,
                        'PATH': virtualenv + '/bin:' + os.environ['PATH']}
        else:
            self.env = {}

        self.virtualenv = virtualenv
        self.ignore_codes = set(ignore_codes)
        self.use_sane_defaults = use_sane_defaults
        self.stream = stream

    @property
    def operative_ignore_codes(self):
        if self.use_sane_defaults:
            return self.ignore_codes ^ self.sane_default_ignore_codes
        else:
            return self.ignore_codes

    @property
    def run_flags(self):
        return ()

    @staticmethod
    def fixup_data(data):
        return data

    @classmethod
    def process_output(cls, line):
        logging.debug(line)
        m = cls.output_matcher.match(line)
        if m:
            fixed_data = dict.fromkeys(('level', 'error_type',
                                        'error_number', 'description',
                                        'filename', 'line_number'),
                                       '')
            fixed_data.update(cls.fixup_data(m.groupdict()))
            print cls.output_format % fixed_data

    def run(self, filename):
        cmdline = [self.command]
        cmdline.extend(self.run_flags)
        cmdline.append(filename)

        env = dict(os.environ, **self.env)
        logging.debug(' '.join(cmdline))
        process = Popen(cmdline, stdout=PIPE, stderr=PIPE, env=env)

        for line in getattr(process, self.stream):
            self.process_output(line)


class PylintRunner(LintRunner):
    """ Run pylint, producing flymake readable output.

    The raw output looks like:
      render.py:49: [C0301] Line too long (82/80)
      render.py:1: [C0111] Missing docstring
      render.py:3: [E0611] No name 'Response' in module 'werkzeug'
      render.py:32: [C0111, render] Missing docstring """

    output_matcher = re.compile(
        r'(?P<filename>[^:]+):'
        r'(?P<line_number>\d+):'
        r'\s*\[(?P<error_type>[WECR])(?P<error_number>[^,]+),'
        r'\s*(?P<context>[^\]]+)\]'
        r'\s*(?P<description>.*)$')

    command = 'pylint'
    sane_default_ignore_codes = set([
        "C0103",  # Naming convention
        "C0111",  # Missing Docstring
        "E1002",  # Use super on old-style class
        "W0232",  # No __init__
        #"I0011",  # Warning locally suppressed using disable-msg
        #"I0012",  # Warning locally suppressed using disable-msg
        #"W0511",  # FIXME/TODO
        #"W0142",  # *args or **kwargs magic.
        "R0904",  # Too many public methods
        "R0903",  # Too few public methods
        "R0201",  # Method could be a function
        ])

    @staticmethod
    def fixup_data(data):
        if data['error_type'].startswith('E'):
            data['level'] = 'ERROR'
        else:
            data['level'] = 'WARNING'
        return data

    @property
    def run_flags(self):
        return ('--output-format', 'parseable',
                '--include-ids', 'y',
                '--reports', 'n',
                '--disable-msg=' + ','.join(self.operative_ignore_codes))


class PycheckerRunner(LintRunner):
    """ Run pychecker, producing flymake readable output.

    The raw output looks like:
      render.py:49: Parameter (maptype) not used
      render.py:49: Parameter (markers) not used
      render.py:49: Parameter (size) not used
      render.py:49: Parameter (zoom) not used """

    command = 'pychecker'

    output_matcher = re.compile(
        r'(?P<filename>[^:]+):'
        r'(?P<line_number>\d+):'
        r'\s+(?P<description>.*)$')

    @staticmethod
    def fixup_data(data):
        #XXX: doesn't seem to give the level
        data['level'] = 'WARNING'
        return data

    @property
    def run_flags(self):
        return '--no-deprecated', '-0186', '--only', '-#0'


class Pep8Runner(LintRunner):
    """ Run pep8.py, producing flymake readable output.

    The raw output looks like:
      spiders/structs.py:3:80: E501 line too long (80 characters)
      spiders/structs.py:7:1: W291 trailing whitespace
      spiders/structs.py:25:33: W602 deprecated form of raising exception
      spiders/structs.py:51:9: E301 expected 1 blank line, found 0 """

    command = 'pep8'
    # sane_default_ignore_codes = set([
    #     'RW29', 'W391',
    #     'W291', 'WO232'])

    output_matcher = re.compile(
        r'(?P<filename>[^:]+):'
        r'(?P<line_number>[^:]+):'
        r'[^:]+:'
        r' (?P<error_number>\w+) '
        r'(?P<description>.+)$')

    @staticmethod
    def fixup_data(data):
        if 'W' in data['error_number']:
            data['level'] = 'WARNING'
        else:
            data['level'] = 'ERROR'

        return data

    @property
    def run_flags(self):
        return '--repeat', '--ignore=' + ','.join(self.ignore_codes)


class TestRunner(LintRunner):
    """ Run unit tests, producing flymake readable output.

    """

    def __init__(self, command, flags, **kwargs):
        self.command = command
        self.flags = flags
        super(TestRunner, self).__init__(**kwargs)

    output_matcher = re.compile(
        r'(?P<filename>[^:]+):'
        r'(?P<line_number>[^:]+): '
        r'In (?P<function>[^:]+): '
        r'(?P<error_number>[^:]+): '
        r'(?P<description>.+)$')

    LEVELS = {'fail': 'ERROR'}

    @staticmethod
    def fixup_data(data):
        data['level'] = TestRunner.LEVELS.get(data['error_number'], 'WARNING')

        return data

    @property
    def run_flags(self):
        return self.flags


def find_config(path):
    if path in ('', '/'):
        module = None
    else:
        try:
            parent_dir = os.path.join(path, '.pyflymakerc')
            module = imp.load_source('config', parent_dir)
        except IOError:
            module = find_config(os.path.split(path)[0])
    return module

if __name__ == '__main__':
    from optparse import OptionParser
    parser = OptionParser()
    parser.add_option("-e", "--virtualenv",
                      dest="virtualenv",
                      default=None,
                      help="virtualenv directory")
    parser.add_option("-i", "--ignore_codes",
                      dest="ignore_codes",
                      default=(),
                      help="error codes to ignore")
    parser.add_option("-d", "--debug",
                      action='store_true',
                      dest="debug",
                      help="print debugging on stderr")
    options, args = parser.parse_args()

    logging.basicConfig(
        level=options.debug and logging.DEBUG or logging.WARNING,
        format='%(levelname)-8s %(message)s')

    config = find_config(args[0])
    virtenv = getattr(config, 'VIRTUALENV', None)
    test_runner_command = getattr(config, 'TEST_RUNNER_COMMAND', None)
    test_runner_flags = getattr(config, 'TEST_RUNNER_FLAGS', [])
    test_runner_output = getattr(config, 'TEST_RUNNER_OUTPUT', 'stdout')

    if options.virtualenv:
        virtenv = options.virtualenv

    if test_runner_command:
        tests = TestRunner(command=test_runner_command,
                           flags=test_runner_flags,
                           stream=test_runner_output,
                           virtualenv=virtenv,
                           ignore_codes=options.ignore_codes)
        tests.run(args[0])

    pylint = PylintRunner(virtualenv=virtenv,
                          ignore_codes=options.ignore_codes)
    pylint.run(args[0])

    pychecker = PycheckerRunner(virtualenv=virtenv,
                                ignore_codes=options.ignore_codes)
    pychecker.run(args[0])

    pep8 = Pep8Runner(virtualenv=virtenv,
                      ignore_codes=options.ignore_codes)
    pep8.run(args[0])
    sys.exit()