chatLZMA-Twitter

chatLZMA-twitter.py

import json
import lzma
import nltk
import random

nltk.download('twitter_samples')

my_filters = [
    {"id": lzma.FILTER_LZMA2, "preset": 9 | lzma.PRESET_EXTREME},
]

lzc = lzma.LZMACompressor(lzma.FORMAT_RAW, filters=my_filters)

reader = nltk.corpus.TwitterCorpusReader('/root/nltk_data/corpora/twitter_samples', '.*\.json')

all_tweets = ''
for tweet in reader.docs():
  all_tweets += json.dumps(tweet['text'])

corp = all_tweets.encode()
out = lzc.compress(corp)
out_end = lzc.flush()

lzd = lzma.LZMADecompressor(lzma.FORMAT_RAW, filters=my_filters)

lzd.decompress(out)
lzd.decompress(out_end[:-50])

i = 0
while True:
  try:
    print(i, '\t', lzd.decompress(random.randbytes(10)).decode(errors="ignore"))
    i += 1
  except Exception as e:
    print(e)
    break