lotusk

from pyspark.sql.types import *
from pyspark.sql.functions import *

#Flatten array of structs and structs
def flatten(df):
   # compute Complex Fields (Lists and Structs) in Schema   
   complex_fields = dict([(field.name, field.dataType)
                             for field in df.schema.fields
                             if type(field.dataType) == ArrayType or  type(field.dataType) == StructType])
   while len(complex_fields)!=0:

lotusk

Update Nvidia driver on CentOS 7.x

Download the latest Nvidia driver on http://www.nvidia.com/drivers

Update the kernel to the latest version

$ yum update

Change to runlevel 3 - multi user mode for the next reboot

lotusk

# ### Interface cuSOLVER PyCUDA

import pycuda.gpuarray as gpuarray
import pycuda.driver as cuda
import pycuda.autoinit
import numpy as np
import scipy.sparse as sp
import ctypes

## Wrap the cuSOLVER cusolverSpDcsrlsvqr() using ctypes