dpla_export_vectors.py

import gensim
import os
import collections
import smart_open
import random
import json
import logging
import numpy as np


logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)


model = gensim.models.doc2vec.Doc2Vec.load('all_docs_simple_model')

file_counter = 0
out = open('vectors_'+str(file_counter)+'.ndjson','w')
for x in range(len(model.docvecs)):
	data = json.dumps({"i":x,"d":model.docvecs.index_to_doctag(x),"v":np.array(model.docvecs[x]).tolist()})
	out.write(data + "\n")
	if x % 5000000 == 0 and x != 0:
		file_counter = file_counter + 1
		out.close()
		out = open('vectors_'+str(file_counter)+'.ndjson','w')
	if x % 10000 == 0:
		print(x)