Skip to content

Instantly share code, notes, and snippets.

@HyukjinKwon
Created July 4, 2017 00:55
Show Gist options
  • Save HyukjinKwon/b1b2c7f65865444c4a8836435100e398 to your computer and use it in GitHub Desktop.
Save HyukjinKwon/b1b2c7f65865444c4a8836435100e398 to your computer and use it in GitHub Desktop.
[PYSPARK] SPARK-19507 codes
from pyspark.sql.types import *
from pyspark.sql import SparkSession
spark = SparkSession \
.builder \
.appName("Python Spark Test") \
.getOrCreate()
def print_message(f):
try:
f()
except Exception as e:
print e.message
schema = StructType([StructField("a", IntegerType())])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame([["a"]], schema))
schema = IntegerType()
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame(["a"], IntegerType()))
schema = StructType([StructField("a", ArrayType(IntegerType()))])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame(["a"], schema))
schema = StructType([StructField("a", ArrayType(IntegerType()))])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame([["a"]], schema))
schema = StructType([StructField("a", ArrayType(IntegerType()))])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame([[["a"]]], schema))
schema = StructType([StructField("a", ArrayType(StructType([StructField("a", IntegerType())])))])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame([[["b"]]], schema))
schema = StructType([StructField("a", MapType(IntegerType(), IntegerType()))])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame([{"a": "1"}], schema))
schema = StructType([StructField("a", MapType(IntegerType(), IntegerType()))])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame([[{"1": "aa"}]], schema))
schema = StructType([StructField("a", MapType(IntegerType(), IntegerType()))])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame([[{1: "aa"}]], schema))
schema = StructType([StructField("a", MapType(IntegerType(), StructType([StructField("a", IntegerType())])))])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame([[{1: ["1"]}]], schema))
schema = StructType([StructField("a", StructType([StructField("b", IntegerType())]))])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame([[["1"]]], schema))
schema = StructType([StructField("a", StructType([StructField("b", StructType([StructField("c", IntegerType())]))]))])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame([[[["1"]]]], schema))
schema = StructType([StructField("a", StructType([StructField("b", StructType([StructField("c", StructType([StructField("b", IntegerType())]))]))]))])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame([[[[["1"]]]]], schema))
schema = StructType([StructField("a", ArrayType(ArrayType(IntegerType())))])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame([[[["a"]]]], schema))
schema = StructType([StructField("a", MapType(StringType(), MapType(IntegerType(), IntegerType())))])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame([[{"a": {"a": "b"}}]], schema))
schema = StructType([StructField("a", MapType(StringType(), MapType(IntegerType(), IntegerType())))])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame([[{"a": {1: "b"}}]], schema))
schema = StructType([StructField("a", StructType([StructField("b", MapType(StringType(), MapType(IntegerType(), IntegerType())))]))])
print
print schema.simpleString()
print_message(lambda: spark.createDataFrame([[[{"a": {1: "b"}}]]], schema))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment