tokenizes Russian text

RusTokenizer

#! /usr/bin/env python
# -*- coding: utf-8 -*-

import nltk
import codecs
from urllib import urlopen

def print_list(mylist):
    '''Print a list containing unicode characters.'''
    print '[' + ', '.join(
        "" + word.encode('utf8') + "" for word in mylist) + ']'

data = codecs.open("masterandmargarita.txt", encoding="utf8")

text = data.read()

tokens = nltk.word_tokenize(text)

print len(tokens)

print len(set(tokens))

print_list(tokens[:200])