| { | |
| "name": "Python 3.8", | |
| "build": { | |
| "dockerfile": "../docker/dev.Dockerfile", | |
| "context": "..", | |
| // Update 'VARIANT' to pick a Python version. Rebuild the container | |
| // if it already exists to update. Available variants: 3, 3.6, 3.7, 3.8 | |
| "args": { "VARIANT": "3.8" } | |
| }, |
| # List hadoop conf while running a pyspark job | |
| hadoopConf = {} | |
| iterator = sc._jsc.hadoopConfiguration().iterator() | |
| while iterator.hasNext(): | |
| prop = iterator.next() | |
| hadoopConf[prop.getKey()] = prop.getValue() | |
| for item in sorted(hadoopConf.items()): |
Define the custom function in the dag :
sc = SlackClient(my_slack_token)
def slack_failed_task(contextDictionary, **kwargs):
base_url = mybasedUrl
task_instance = contextDictionary.get('task_instance')
log_url = '{task_instance.log_url}'.format(**locals())
failed_alert = sc.api_call('chat.postMessage',