Skip to content

Instantly share code, notes, and snippets.

@htjelsma
htjelsma / custom_sql_in_spark.py
Last active October 29, 2019 08:52
custom_sql_in_spark.sql
#Set up a JDBC connection to your database
#Get the credentials from the Keyvault that a scope was defined for using the
#Databricks CLI. See: https://docs.azuredatabricks.net/security/secrets/secret-scopes.html
#alternatively, hard-code the credentials here:
jdbcUrl = dbutils.secrets.get(scope="JDBC", key="url")
jdbcUsername = dbutils.secrets.get(scope="JDBC", key="username")
jdbcPassword = dbutils.secrets.get(scope="JDBC", key="password")
#create an object with the credentials that we can pass on to the spark dataframe reader
rc = spark.read.jdbc(url=jdbcUrl, table='(select 1 as nr) as t', properties=connectionProperties).count()
connectionProperties['sessionInitStatement'] = '<insert custom SQL code here>'
#Get the credentials from the Keyvault that a scope was defined for using the
#Databricks CLI. See: https://docs.azuredatabricks.net/security/secrets/secret-scopes.html
#alternatively, hard-code the credentials here:
jdbcUrl = dbutils.secrets.get(scope="JDBC", key="url")
jdbcUsername = dbutils.secrets.get(scope="JDBC", key="username")
jdbcPassword = dbutils.secrets.get(scope="JDBC", key="password")
#create an object with the credentials that we can pass on to the spark dataframe reader
connectionProperties = {
"user" : jdbcUsername,