k-means clustering using spacy for vectorization

file_op.py

import csv


def read_csv_file():
    with open('/Users/shinysuresh/PycharmProjects/mlprojects/kmeans_spacy/Questions_bkup.csv', 'r') as file:
        reader = csv.reader(file)
        #print(reader)
        records = ['\n'.join(row) for row in reader]
    return records


def write_csv_file(rows):
    with open('/Users/shinysuresh/PycharmProjects/mlprojects/kmeans_spacy/question_clusters_22.csv', 'w', newline='') as file:
        writer = csv.writer(file)
        writer.writerows(rows)


'''def copy_csv():
    with open('Questions.csv', 'a') as output:
        writer = csv.writer(output)
        with open('Questions2.csv', 'r') as input:
            reader = csv.reader(input)
            for row in reader:
                writer.writerow(row)'''


'''if __name__ == "__main__":
    read_csv_file()'''

k-means_spacy

import spacy
import collections
import string
from nltk import word_tokenize
from nltk.corpus import stopwords
from sklearn.cluster import KMeans
from file_op import read_csv_file, write_csv_file


nlp = spacy.load('en_core_web_sm')


def clean_string(inp_str):
    st = str.maketrans('', '', string.punctuation)
    return inp_str.translate(st).lower().strip().split()


def convert_to_vec(question):
    tokens = clean_string(question)
    tokens = [t for t in tokens if t not in stopwords.words('english')]
    nlp = spacy.load('en')
    doc = nlp(str(question))  
    return(doc.vector)


def cluster_questions(questions, nb_of_clusters=5):
    question_vectors = [convert_to_vec(question) for question in questions]
    kmeans = KMeans(n_clusters=nb_of_clusters)
    kmeans.fit(question_vectors)
    clusters = collections.defaultdict(list)
    for i, label in enumerate(kmeans.labels_):
        clusters[label].append(i)
    return(dict(clusters))


if __name__ == "__main__":
    questions = read_csv_file()
    nclusters= 22
    clusters=cluster_questions(questions, nclusters)
    question_clusters=[]
    for cluster in range(nclusters):
        question_clusters.append(["Cluster"+str(cluster+1)+":"])
        for i,sentence in enumerate(clusters[cluster]):
            question_clusters.append([str(questions[sentence])])
    write_csv_file(question_clusters)

question_clusters_15.csv

question_clusters_22.csv

question_clusters_5.csv

Questions_bkup.csv