Skip to content

Instantly share code, notes, and snippets.

View japanese_tokenizer.py
import re
from typing import Any, Dict, List, Text
from rasa.nlu.components import Component
from rasa.nlu.config import RasaNLUModelConfig
from rasa.nlu.tokenizers import Token, Tokenizer
from rasa.nlu.training_data import Message, TrainingData
class JapaneseTokenizer(Tokenizer, Component):