Set up spylon-kernel notebook with Spark

spylon-kernal.md

%%init_spark
# Configure the location of the mesos master and spark distribution on HDFS
# launcher.jars = ["/some/local/path/to/a/file.jar"]
launcher.packages = ["org.elasticsearch:elasticsearch-hadoop:6.6.0",
                     "org.apache.hadoop:hadoop-aws:2.7.5",
                     "org.apache.hadoop:hadoop-common:2.7.5",
                     "com.amazonaws:aws-java-sdk:1.7.4"]
launcher.master = "k8s://kubernetes.default"
launcher.conf.set("spark.submit.deployMode", "client")
launcher.conf.set("spark.driver.port", "7077")
launcher.conf.set("spark.kubernetes.container.image", "ianchen06/spark-py:v2.4.3")
launcher.conf.set("spark.executor.instances", "2")
launcher.conf.set("spark.app.name", "ian")
launcher.conf.set("spark.kubernetes.namespace", "jupyterhub")
launcher.conf.set("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
launcher.conf.set("spark.hadoop.fs.s3a.endpoint", "http://<host>:9000")
launcher.conf.set("spark.hadoop.fs.s3a.access.key", "<key>")
launcher.conf.set("spark.hadoop.fs.s3a.secret.key", "<secret>")
launcher.conf.set("spark.sql.catalogImplementation", "hive")
launcher.conf.set("hive.metastore.uris", "thrift://<host>:9083")
launcher.conf.set("spark.sql.hive.metastore.version", "2.3.3")
launcher.conf.set("spark.sql.hive.metastore.jars", "maven")