Remove unk chars in English and Vietnamese document

clean_envi_doc.py

import re

CHARACTERS = "0123456789aAàÀảẢãÃáÁạẠăĂằẰẳẲẵẴắẮặẶâÂầẦẩẨẫẪấẤậẬbBcCdDđĐeEèÈẻẺẽẼéÉẹẸêÊềỀểỂễỄếẾệỆfFgGhHiIìÌỉỈĩĨíÍị" \
             "ỊjJkKlLmMnNoOòÒỏỎõÕóÓọỌôÔồỒổỔỗỖốỐộỘơƠờỜởỞỡỠớỚợỢpPqQrRsStTuUùÙủỦũŨúÚụỤưƯừỪửỬữỮứỨựỰvVwWxXyYỳỲỷỶỹỸýÝỵỴzZ"
PUNCTUATION = ".,?!@%~`#$^&*()-_+=[]{}\|:;\"'<>/"
ALL_CHARS = CHARACTERS + PUNCTUATION
WORD_NORMALIZER = re.compile(r"[^ {}]".format(re.escape(ALL_CHARS)))

def remove_unk_char(text):
    return WORD_NORMALIZER.sub(' ', text)

def strip_space(text):
    return re.sub(r'\s+', ' ', text.strip())

def format_text(text, remove_threshold=0.2):
    text = strip_space(text)
    root_len = len(text)
    text = remove_unk_char(text)
    text = strip_space(text)
    if 1 - len(text) / root_len > remove_threshold:
        return ""
    return text