bpe_1.py

#importing library
import pandas as pd

#reading .txt file
text = pd.read_csv("sample.txt",header=None)

#converting a dataframe into a single list 
corpus=[]
for row in text.values:
    tokens = row[0].split(" ")
    for token in tokens:
        corpus.append(token)