Pyspark MLLib || GMM clustering Error

GMM_Error_demo.ipynb

#!/usr/bin/env python
# coding: utf-8

# In[1]:


import pyspark
import sys
print("python version = {}".format(sys.version))
print("pyspark version = {}".format(pyspark.__version__))


# In[2]:


from pyspark.sql.types import DoubleType, StringType, IntegerType, ArrayType
from pyspark.ml.clustering import GaussianMixture
from pyspark.sql import SQLContext
from pyspark.sql.session import SparkSession
from pyspark.mllib.linalg import Vectors, VectorUDT, DenseVector
from pyspark.sql.functions import udf

sc = SparkSession.builder.appName('GMM_Demo').master("local[*]").getOrCreate()
sql = SQLContext(sc)
df = sc.read.csv("GMM_demo_data.csv", header = True)


# In[3]:


# new_schema = ArrayType(DoubleType(), containsNull=False)
new_schema = ArrayType(DoubleType(), containsNull=False)

string_parse = udf(lambda x: [float(elem.replace('[','')
                                        .replace(']','')
                                        .strip()) for elem in x.split(",")], new_schema)

df = df.withColumn('features', string_parse('pcaFeatures'))


# In[45]:


model = GaussianMixture(k=7, seed = 1234, featuresCol = 'features',
                        predictionCol='predictionCol', probabilityCol = 'probabilityCol')

output = model.fit(df).transform(df)


# In[50]:


output.select('probabilityCol').toPandas()