language_identification_trainvaltest.py

# Splitting into train, val and test set -- 80-10-10 split

# First, an 80-20 split
train_df, val_test_df = train_test_split(df, test_size = 0.2, random_state = 3)

# Then split the 20% into half
val_df, test_df = train_test_split(val_test_df, test_size = 0.5, random_state = 3)

Xtrain = train_bow.toarray()
ytrain = train_df[languages]

Xval = val_bow.toarray()
yval = val_df[languages]

ytest = test_df[languages]
Xtest = test_bow.toarray()