rnn19

def tockenize(X_train,X_test):
  """
  bow encoding
  """
  word_list = []
  for sent in X_train:
    for word in sent.split():
      word_list.append(word)

  corpus = Counter(word_list)
  # sorting on the basis of most common words
  corpus_ = sorted(corpus,key=corpus.get,reverse=True)
  # creating a dict
  onehot_dict = {w:i+1 for i,w in enumerate(corpus_)}

  train_vec = []
  test_vec = []
  for sent in X_train:
      train_vec.append([onehot_dict[word] for word in sent.split() if word in onehot_dict.keys()])

  for sent in X_test:
      test_vec.append([onehot_dict[word] for word in sent.split() if word in onehot_dict.keys()])

  return train_vec,test_vec,corpus_
  


essay_train = X_train['essay'].values
essay_test = X_test['essay'].values  
# encoding
essay_train_p,essay_test_p,corpus = tockenize(essay_train,essay_test)
print(len(corpus))