This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# change datatype of 'movie_id' | |
meta['movie_id'] = meta['movie_id'].astype(str) | |
# merge meta with movies | |
movies = pd.merge(movies, meta[['movie_id', 'movie_name', 'genre']], on = 'movie_id') | |
movies.head() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# an empty list | |
genres = [] | |
# extract genres | |
for i in movies['genre']: | |
genres.append(list(json.loads(i).values())) | |
# add to 'movies' dataframe | |
movies['genre_new'] = genres |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
all_genres = nltk.FreqDist(all_genres) | |
# create dataframe | |
all_genres_df = pd.DataFrame({'Genre': list(all_genres.keys()), | |
'Count': list(all_genres.values())}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
g = all_genres_df.nlargest(columns="Count", n = 50) | |
plt.figure(figsize=(12,15)) | |
ax = sns.barplot(data=g, x= "Count", y = "Genre") | |
ax.set(ylabel = 'Count') | |
plt.show() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# function for text cleaning | |
def clean_text(text): | |
# remove backslash-apostrophe | |
text = re.sub("\'", "", text) | |
# remove everything except alphabets | |
text = re.sub("[^a-zA-Z]"," ",text) | |
# remove whitespaces | |
text = ' '.join(text.split()) | |
# convert text to lowercase | |
text = text.lower() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def freq_words(x, terms = 30): | |
all_words = ' '.join([text for text in x]) | |
all_words = all_words.split() | |
fdist = nltk.FreqDist(all_words) | |
words_df = pd.DataFrame({'word':list(fdist.keys()), 'count':list(fdist.values())}) | |
# selecting top 20 most frequent words | |
d = words_df.nlargest(columns="count", n = terms) | |
# visualize words and frequencies |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from nltk.corpus import stopwords | |
stop_words = set(stopwords.words('english')) | |
# function to remove stopwords | |
def remove_stopwords(text): | |
no_stopword_text = [w for w in text.split() if not w in stop_words] | |
return ' '.join(no_stopword_text) | |
movies_new['clean_plot'] = movies_new['clean_plot'].apply(lambda x: remove_stopwords(x)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from sklearn.preprocessing import MultiLabelBinarizer | |
multilabel_binarizer = MultiLabelBinarizer() | |
multilabel_binarizer.fit(movies_new['genre_new']) | |
# transform target variable | |
y = multilabel_binarizer.transform(movies_new['genre_new']) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# split dataset into training and validation set | |
xtrain, xval, ytrain, yval = train_test_split(movies_new['clean_plot'], y, test_size=0.2, random_state=9) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def infer_tags(q): | |
q = clean_text(q) | |
q = remove_stopwords(q) | |
q_vec = tfidf_vectorizer.transform([q]) | |
q_pred = clf.predict(q_vec) | |
return multilabel_binarizer.inverse_transform(q_pred) |