pylint_utf8_issuefix.py.patch

# HG changeset patch
# User Alexander Artemenko <svetlyak.40wt@gmail.com>
# Date 1269880368 -14400
# Node ID 71abc0ecd0283583cde79914c65a92712c8cda53
# Parent  cdd571901fea51f7677d6f744ea9cd2d82c57d68
Fixes issue 4683. Non-ASCII characters count double if utf8 encode.

diff -r cdd571901fea -r 71abc0ecd028 checkers/format.py
--- a/checkers/format.py        Mon Mar 29 11:27:19 2010 +0200
+++ b/checkers/format.py        Mon Mar 29 20:32:48 2010 +0400
@@ -31,6 +31,7 @@
 
 from pylint.interfaces import IRawChecker, IASTNGChecker
 from pylint.checkers import BaseRawChecker
+from pylint.checkers.misc import guess_encoding, is_ascii
 
 MSGS = {
     'C0301': ('Line too long (%s/%s)',
@@ -177,6 +178,25 @@
         BaseRawChecker.__init__(self, linter)
         self._lines = None
         self._visited_lines = None
+
+    def process_module(self, stream):
+        """extracts encoding from the stream and
+        decodes each line, so that international
+        text's lenght properly calculated.
+        """
+        data = stream.read()
+        line_generator = stream.readline
+
+        ascii, lineno = is_ascii(data)
+        if not ascii:
+            encoding = guess_encoding(data)
+            if encoding is not None:
+                line_generator = lambda: stream.readline().decode(encoding)
+
+        del data
+
+        stream.seek(0)
+        self.process_tokens(tokenize.generate_tokens(line_generator))
 
     def new_line(self, tok_type, line, line_num, junk):
         """a new line has been encountered, process it if necessary"""
diff -r cdd571901fea -r 71abc0ecd028 test/input/func_toolonglines_utf8.py
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/test/input/func_toolonglines_utf8.py      Mon Mar 29 20:32:48 2010 +0400
@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+""" Эта строчка дейстительно оооооооооооооооооооооооооооооооооооооооооооооооочень длинная!"""
+
+def blah():
+    """ А вот эта строка сравнительно короткая."""
+    pass
+
+__revision__ = ''
diff -r cdd571901fea -r 71abc0ecd028 test/messages/func_toolonglines_utf8.txt
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/test/messages/func_toolonglines_utf8.txt  Mon Mar 29 20:32:48 2010 +0400
@@ -0,0 +1,1 @@
+C:  2: Line too long (93/80)