gistfile1.txt

export SPARK_HOME= /data/local/opt/spark-2.4.3-bin-hadoop2.7
cd $SPARK_HOME

## scala v 2.11 (!!)

./bin/spark-shell -v  --master yarn-client --num-executors 20 --driver-memory 2g --executor-memory 2g \
--jars /tmp/bdg-sequila-acc_2.11-0.1-spark-2.4.3-SNAPSHOT-assembly.jar \
--conf spark.sql.extensions=org.biodatageeks.sequila.spark.BdgExtensions \
--conf spark.hadoop.yarn.timeline-service.enabled=false \
--conf spark.hadoop.hive.metastore.uris=thrift://cdh01.cl.ii.pw.edu.pl:9083 \
--conf spark.hadoop.yarn.timeline-service.enabled=false \
--conf spark.driver.extraJavaOptions=-Dhdp.version=3.1.0.0-78 \
--conf spark.yarn.am.extraJavaOptions=-Dhdp.version=3.1.0.0-78 \
--conf spark.hadoop.metastore.catalog.default=hive \
--conf spark.jars.ivy=/data/local/cache/ivy2/repository/





--packages org.biodatageeks:bdg-sequila-acc_2.12:0.1-spark-2.4.3-SNAPSHOT \