dpla_build_annoy_index.py

import json
from annoy import AnnoyIndex

files = ['vectors_0.ndjson','vectors_1.ndjson','vectors_2.ndjson','vectors_3.ndjson']
t = AnnoyIndex(100)  # Length of item vector that will be indexed

lookup = {}
counter = 0
for f in files:
	print(f)
	with open(f,'r') as file:
		for line in file:
			counter += 1
			data = json.loads(line)
			t.add_item(data['i'], data['v'])
			if counter % 50000 ==0:
				 print(counter, end='')
				 print('\r', end='')

t.build(50) # 50 trees
t.save('index_50tree.ann')