reproduce of tokenizers hang on encode_batch

bpe_hang.py

from multiprocessing import Process
import os
from tokenizers.implementations import ByteLevelBPETokenizer
import tokenizers

print(tokenizers.__version__)

# works:
tok = ByteLevelBPETokenizer()
print(tok.encode_batch(['ala']))
print(tok.encode_batch(['ala', 'kot']))

def encode(name):
    tok = ByteLevelBPETokenizer()
    print("single text")
    print(tok.encode_batch(['ala']))
    print(tok.encode_batch(['ala', 'kot']))

p = Process(target=encode, args=('ala',))
p.start()
p.join()

output.log

0.6.0
[Encoding(num_tokens=0, attributes=[ids, type_ids, tokens, offsets, attention_mask, special_tokens_mask, overflowing, original_str, normalized_str])]
[Encoding(num_tokens=0, attributes=[ids, type_ids, tokens, offsets, attention_mask, special_tokens_mask, overflowing, original_str, normalized_str]), Encoding(num_tokens=0, attributes=[ids, type_ids, tokens, offsets, attention_mask, special_tokens_mask, overflowing, original_str, normalized_str])]
single text
[Encoding(num_tokens=0, attributes=[ids, type_ids, tokens, offsets, attention_mask, special_tokens_mask, overflowing, original_str, normalized_str])]

... hangs here