The function to do the Tokenization

tokenization.py

def _tokenize(self):
  work_sentence = self.raw
  for punctuation in self._punctuations:
      work_sentence = work_sentence.replace(punctuation,
        " "+punctuation+" ")
  for delimiter in self._token_boundaries:
      work_sentence = work_sentence.replace(delimiter,
        self._delimiter_token)
  self.tokens = [x.strip() for x in work_sentence.split(self._delimiter_token) if x != '']