Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
{
"name": "Execute Databricks Job using MI with parms",
"properties": {
"description": "Databricks Application ID: 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d",
"activities": [
{
"name": "Execute Jobs API",
"type": "WebActivity",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"url": {
"value": "@concat('https://',pipeline().parameters.DatabricksWorkspaceID,'.azuredatabricks.net/api/2.1/jobs/run-now')",
"type": "Expression"
},
"connectVia": {
"referenceName": "AutoResolveIntegrationRuntime",
"type": "IntegrationRuntimeReference"
},
"method": "POST",
"body": {
"value": "@if(empty(pipeline().parameters.Parameters), \nconcat('{\"job_id\":',pipeline().parameters.JobID,'}'), \nconcat('{\"job_id\":',pipeline().parameters.JobID,\n',\"notebook_params\":', pipeline().parameters.Parameters,'}')\n)",
"type": "Expression"
},
"authentication": {
"type": "MSI",
"resource": "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d"
}
}
},
{
"name": "Wait Until Job Completes",
"type": "Until",
"dependsOn": [
{
"activity": "Execute Jobs API",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"expression": {
"value": "@not(equals(variables('JobStatus'),'Running'))",
"type": "Expression"
},
"activities": [
{
"name": "Check Job Run API",
"type": "WebActivity",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"url": {
"value": "@concat('https://',pipeline().parameters.DatabricksWorkspaceID,'.azuredatabricks.net/api/2.1/jobs/runs/get?run_id=',activity('Execute Jobs API').output.run_id)",
"type": "Expression"
},
"connectVia": {
"referenceName": "AutoResolveIntegrationRuntime",
"type": "IntegrationRuntimeReference"
},
"method": "GET",
"body": {
"job_id": 3895
},
"authentication": {
"type": "MSI",
"resource": "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d"
}
}
},
{
"name": "Set Job Status",
"type": "SetVariable",
"dependsOn": [
{
"activity": "Check Job Run API",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"variableName": "JobStatus",
"value": {
"value": "@if(\nor(\nequals(activity('Check Job Run API').output.state.life_cycle_state, 'PENDING'), equals(activity('Check Job Run API').output.state.life_cycle_state, 'RUNNING')\n), \n'Running',\nactivity('Check Job Run API').output.state.result_state\n)",
"type": "Expression"
}
}
},
{
"name": "Wait to Recheck API",
"type": "Wait",
"dependsOn": [
{
"activity": "Set Job Status",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": {
"value": "@pipeline().parameters.WaitSeconds",
"type": "Expression"
}
}
}
],
"timeout": "7.00:00:00"
}
}
],
"parameters": {
"JobID": {
"type": "string",
"defaultValue": "4560"
},
"DatabricksWorkspaceID": {
"type": "string",
"defaultValue": "adb-5343834423590926.6"
},
"WaitSeconds": {
"type": "int",
"defaultValue": 30
},
"Parameters": {
"type": "string",
"defaultValue": {
"ETL_ID": "60"
}
}
},
"variables": {
"JobStatus": {
"type": "String",
"defaultValue": "Running"
}
},
"folder": {
"name": "Jobs API"
},
"annotations": [],
"lastPublishTime": "2021-10-26T17:26:15Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment