Customize word tokenization: add and remove words from trie

modify_trie.py

!pip install pythainlp
from pythainlp import word_tokenize
from pythainlp.tokenize import DEFAULT_DICT_TRIE as trie
# default behavior
print(word_tokenize('ฝนตกทั่วฟ้า'))  # ['ฝนตก', 'ทั่ว', 'ฟ้า']
# modify behavior
trie.remove('ฝนตก')
trie.add('ทั่วฟ้า')
word_tokenize('ฝนตกทั่วฟ้า')  # ['ฝน', 'ตก', 'ทั่วฟ้า']

replace_trie.py

from pythainlp.tokenize import word_tokenize, newmm
from pythainlp.corpus import ttc
from pythainlp.util import Trie
words = [w for w,_ in ttc.word_freqs()]
newmm.DEFAULT_WORD_DICT_TRIE = Trie(words)
word_tokenize('ฝนตกทั่วฟ้า')   # ['ฝน', 'ตก', 'ทั่วฟ้า']