make_english_small.py

import json

# word_frequency.txt contains one word per line, most common words first
with open("word_frequency.txt","r",encoding="utf-8") as f: words = f.read().splitlines()

wrds = set()
wrds_list = []
wrds_learn = []

for word in words:
	for L in range(len(word)):
		wrd = word[0:L+1] # hello -> h, he, hel, hell, hello
		if wrd not in wrds:
			wrds.add(wrd)
			wrds_list.append(wrd)
			wrds_learn.append(wrd + " -> " + word)
			break


with open("wrds.txt",  "w", encoding="utf-8") as f: f.write("\n".join(wrds_learn))
with open("wrds.json", "w", encoding="utf-8") as f: json.dump(wrds_learn, f)