Last active
March 7, 2021 20:19
-
-
Save itayB/971a4a1d9d7a5bd91b5df385dd680c94 to your computer and use it in GitHub Desktop.
Spark session creation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "welcome-pressing", | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"from pyspark import SparkConf\n", | |
"from pyspark.sql import SparkSession\n", | |
"\n", | |
"\n", | |
"config = {\n", | |
" \"spark.kubernetes.namespace\": \"spark\",\n", | |
" \"spark.kubernetes.container.image\": \"itayb/spark:3.1.1-hadoop-3.2.0-aws\",\n", | |
" \"spark.executor.instances\": \"2\",\n", | |
" \"spark.executor.memory\": \"1g\",\n", | |
" \"spark.executor.cores\": \"1\",\n", | |
" \"spark.driver.blockManager.port\": \"7777\",\n", | |
" \"spark.driver.port\": \"2222\",\n", | |
" \"spark.driver.host\": \"jupyter.spark.svc.cluster.local\",\n", | |
" \"spark.driver.bindAddress\": \"0.0.0.0\",\n", | |
" \"spark.hadoop.fs.s3a.endpoint\": \"localstack.kube-system.svc.cluster.local:4566\",\n", | |
" \"spark.hadoop.fs.s3a.connection.ssl.enabled\": \"false\",\n", | |
" \"spark.hadoop.fs.s3a.path.style.access\": \"true\",\n", | |
" \"spark.hadoop.fs.s3a.impl\": \"org.apache.hadoop.fs.s3a.S3AFileSystem\",\n", | |
" \"spark.hadoop.com.amazonaws.services.s3.enableV4\": \"true\",\n", | |
" \"spark.hadoop.fs.s3a.aws.credentials.provider\": \"org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider\",\n", | |
"}\n", | |
"\n", | |
"def get_spark_session(app_name: str, conf: SparkConf):\n", | |
" conf.setMaster(\"k8s://https://kubernetes.default.svc.cluster.local\")\n", | |
" for key, value in config.items():\n", | |
" conf.set(key, value) \n", | |
" return SparkSession.builder.appName(app_name).config(conf=conf).getOrCreate()" | |
] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python 3", | |
"language": "python", | |
"name": "python3" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.7.3" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 5 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment