demo_tokenize_time.py

import time

import mwapi
from deltas.tokenizers import wikitext_split

'''text = """
This is a sentence [[derp|link]].

Here is another paragraph with the number 10.
"""'''

session = mwapi.Session("https://en.wikipedia.org")
doc = session.get(action="query", prop="revisions",
                  titles="Alan Turing", rvprop="content", rvslots="main",
                  formatversion=2)
text = doc['query']['pages'][0]['revisions'][0]['slots']['main']['content']

start = time.time()
for i in range(100):
    list(wikitext_split.tokenize(text))
print("We can process", 1/((time.time() - start)/100), "Alan Turing's per second")

This is some text 古池や蛙飛び込む水の音.

["This", " ", "is", " ", "some", " ", "text", " ", "古池や蛙飛び込む水の音", "."]

["This", " ", "is", " ", "some", " ", "text", " ", "古池や", "蛙飛び", "込む水", "の音", "."]

I think we can block CJK together but putting a "+" at the end of the CJK definition here: https://github.com/halfak/deltas/blob/master/deltas/tokenizers/wikitext_split.py#L45

This just demos tokenization

import time

import mwapi
from deltas.tokenizers import wikitext_split

session = mwapi.Session("https://en.wikipedia.org")
doc = session.get(action="query", prop="revisions",
                  titles="Aaron Halfaker", rvprop="content", rvslots="main",
                  formatversion=2)
text = doc['query']['pages'][0]['revisions'][0]['slots']['main']['content']

for token in wikitext_split.tokenize(text):
  print(repr(token))

@5uperpalo