Created
February 3, 2022 16:54
-
-
Save furlong46/b0ac53e36ef5f487c83de20d078af41c to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "Execute Databricks Job using MI with parms", | |
"properties": { | |
"description": "Databricks Application ID: 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d", | |
"activities": [ | |
{ | |
"name": "Execute Jobs API", | |
"type": "WebActivity", | |
"dependsOn": [], | |
"policy": { | |
"timeout": "7.00:00:00", | |
"retry": 0, | |
"retryIntervalInSeconds": 30, | |
"secureOutput": false, | |
"secureInput": false | |
}, | |
"userProperties": [], | |
"typeProperties": { | |
"url": { | |
"value": "@concat('https://',pipeline().parameters.DatabricksWorkspaceID,'.azuredatabricks.net/api/2.1/jobs/run-now')", | |
"type": "Expression" | |
}, | |
"connectVia": { | |
"referenceName": "AutoResolveIntegrationRuntime", | |
"type": "IntegrationRuntimeReference" | |
}, | |
"method": "POST", | |
"body": { | |
"value": "@if(empty(pipeline().parameters.Parameters), \nconcat('{\"job_id\":',pipeline().parameters.JobID,'}'), \nconcat('{\"job_id\":',pipeline().parameters.JobID,\n',\"notebook_params\":', pipeline().parameters.Parameters,'}')\n)", | |
"type": "Expression" | |
}, | |
"authentication": { | |
"type": "MSI", | |
"resource": "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d" | |
} | |
} | |
}, | |
{ | |
"name": "Wait Until Job Completes", | |
"type": "Until", | |
"dependsOn": [ | |
{ | |
"activity": "Execute Jobs API", | |
"dependencyConditions": [ | |
"Succeeded" | |
] | |
} | |
], | |
"userProperties": [], | |
"typeProperties": { | |
"expression": { | |
"value": "@not(equals(variables('JobStatus'),'Running'))", | |
"type": "Expression" | |
}, | |
"activities": [ | |
{ | |
"name": "Check Job Run API", | |
"type": "WebActivity", | |
"dependsOn": [], | |
"policy": { | |
"timeout": "7.00:00:00", | |
"retry": 0, | |
"retryIntervalInSeconds": 30, | |
"secureOutput": false, | |
"secureInput": false | |
}, | |
"userProperties": [], | |
"typeProperties": { | |
"url": { | |
"value": "@concat('https://',pipeline().parameters.DatabricksWorkspaceID,'.azuredatabricks.net/api/2.1/jobs/runs/get?run_id=',activity('Execute Jobs API').output.run_id)", | |
"type": "Expression" | |
}, | |
"connectVia": { | |
"referenceName": "AutoResolveIntegrationRuntime", | |
"type": "IntegrationRuntimeReference" | |
}, | |
"method": "GET", | |
"body": { | |
"job_id": 3895 | |
}, | |
"authentication": { | |
"type": "MSI", | |
"resource": "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d" | |
} | |
} | |
}, | |
{ | |
"name": "Set Job Status", | |
"type": "SetVariable", | |
"dependsOn": [ | |
{ | |
"activity": "Check Job Run API", | |
"dependencyConditions": [ | |
"Succeeded" | |
] | |
} | |
], | |
"userProperties": [], | |
"typeProperties": { | |
"variableName": "JobStatus", | |
"value": { | |
"value": "@if(\nor(\nequals(activity('Check Job Run API').output.state.life_cycle_state, 'PENDING'), equals(activity('Check Job Run API').output.state.life_cycle_state, 'RUNNING')\n), \n'Running',\nactivity('Check Job Run API').output.state.result_state\n)", | |
"type": "Expression" | |
} | |
} | |
}, | |
{ | |
"name": "Wait to Recheck API", | |
"type": "Wait", | |
"dependsOn": [ | |
{ | |
"activity": "Set Job Status", | |
"dependencyConditions": [ | |
"Succeeded" | |
] | |
} | |
], | |
"userProperties": [], | |
"typeProperties": { | |
"waitTimeInSeconds": { | |
"value": "@pipeline().parameters.WaitSeconds", | |
"type": "Expression" | |
} | |
} | |
} | |
], | |
"timeout": "7.00:00:00" | |
} | |
} | |
], | |
"parameters": { | |
"JobID": { | |
"type": "string", | |
"defaultValue": "4560" | |
}, | |
"DatabricksWorkspaceID": { | |
"type": "string", | |
"defaultValue": "adb-5343834423590926.6" | |
}, | |
"WaitSeconds": { | |
"type": "int", | |
"defaultValue": 30 | |
}, | |
"Parameters": { | |
"type": "string", | |
"defaultValue": { | |
"ETL_ID": "60" | |
} | |
} | |
}, | |
"variables": { | |
"JobStatus": { | |
"type": "String", | |
"defaultValue": "Running" | |
} | |
}, | |
"folder": { | |
"name": "Jobs API" | |
}, | |
"annotations": [], | |
"lastPublishTime": "2021-10-26T17:26:15Z" | |
}, | |
"type": "Microsoft.DataFactory/factories/pipelines" | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment