shihono

shihono

import difflib
import unicodedata

def get_char_width_list(text):
    """text の文字ごとの幅をリストで返す
    半角の場合は1, 全角の場合は2
    """
    result = []
    for c in text:
        if unicodedata.east_asian_width(c) in ["F", "W"]:

shihono

shihono

shihono

shihono

shihono

shihono

shihono

import lzma
import sys
import glob

def load_xz_ngram_file(n=2):
    ngram_dict = {}
    ngram_freq = {}
    files = glob.glob("/path/to/nwc2010-ngrams/word/over999/{}gms/*.xz".format(n))
    for file in files:
        print(file)

shihono

from sklearn.base import BaseEstimator
import MeCab


class OwakatiTokenizer(BaseEstimator):
    def __init__(self, dicdir=None, rcfile=None):
        """
        :param dicdir: system dicdir `-d`
        :param rcfile: resource file `-r`
        """