Created
August 10, 2022 15:37
-
-
Save furlong46/1a539b748773e2dbea05d447cadd050d to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "name": "Execute Databricks DLT Pipeline using MI", | |
| "properties": { | |
| "description": "Databricks Application ID: 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d\nDoc Page: https://docs.microsoft.com/en-us/azure/databricks/data-engineering/delta-live-tables/delta-live-tables-workflows#azure-data-factory", | |
| "activities": [ | |
| { | |
| "name": "Update DLT Pipeline", | |
| "type": "WebActivity", | |
| "dependsOn": [], | |
| "policy": { | |
| "timeout": "7.00:00:00", | |
| "retry": 0, | |
| "retryIntervalInSeconds": 30, | |
| "secureOutput": false, | |
| "secureInput": false | |
| }, | |
| "userProperties": [], | |
| "typeProperties": { | |
| "url": { | |
| "value": "@concat('https://',pipeline().parameters.DatabricksWorkspaceID,'.azuredatabricks.net/api/2.0/pipelines/', pipeline().parameters.DLTID)", | |
| "type": "Expression" | |
| }, | |
| "connectVia": { | |
| "referenceName": "AutoResolveIntegrationRuntime", | |
| "type": "IntegrationRuntimeReference" | |
| }, | |
| "method": "PUT", | |
| "body": { | |
| "value": "@json(concat('{\n \"id\": \"ab387469-676c-402c-881d-0ba453b4b708\",\n \"clusters\": [\n {\n \"label\": \"default\",\n \"num_workers\": 2\n }\n ],\n \"development\": true,\n \"continuous\": false,\n \"edition\": \"advanced\",\n \"photon\": true,\n \"libraries\": [\n {\n \"notebook\": {\n \"path\": \"/Repos/leo.furlong@databricks.com/quickstart-labs/ADBQuickStartLabs/DLT Demo/DLT Code\"\n }\n }\n ],\n \"name\": \"QuickStart Labs DLT ADLS\",\n \"storage\": \"abfss://datalake@lafadlspltest.dfs.core.windows.net/dlt_demo_adls/\",\n \"configuration\": {\n \"pipelines.applyChangesPreviewEnabled\": \"true\",\n \"spark.hadoop.fs.azure.account.auth.type.lafadlspltest.dfs.core.windows.net\": \"OAuth\",\n \"spark.hadoop.fs.azure.account.oauth2.client.endpoint.lafadlspltest.dfs.core.windows.net\": \"https://login.microsoftonline.com/9f37a392-f0ae-4280-9796-f1864a10effc/oauth2/token\",\n \"spark.hadoop.fs.azure.account.oauth.provider.type.lafadlspltest.dfs.core.windows.net\": \"org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider\",\n \"spark.hadoop.fs.azure.account.oauth2.client.id.lafadlspltest.dfs.core.windows.net\": \"{{secrets/laf-secret-scope/sp-id}}\",\n \"spark.hadoop.fs.azure.account.oauth2.client.secret.lafadlspltest.dfs.core.windows.net\": \"{{secrets/laf-secret-scope/sp-secrete}}\",\n \"Test\": \"Value\",\n \"TestADF\": \"',pipeline().parameters.ADFTestValue,'\"\n },\n \"target\": \"dlt_demo_adls\"\n}'))", | |
| "type": "Expression" | |
| }, | |
| "authentication": { | |
| "type": "MSI", | |
| "resource": "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d" | |
| } | |
| } | |
| }, | |
| { | |
| "name": "Execute DLT Pipeline", | |
| "type": "WebActivity", | |
| "dependsOn": [ | |
| { | |
| "activity": "Update DLT Pipeline", | |
| "dependencyConditions": [ | |
| "Succeeded" | |
| ] | |
| } | |
| ], | |
| "policy": { | |
| "timeout": "7.00:00:00", | |
| "retry": 0, | |
| "retryIntervalInSeconds": 30, | |
| "secureOutput": false, | |
| "secureInput": false | |
| }, | |
| "userProperties": [], | |
| "typeProperties": { | |
| "url": { | |
| "value": "@concat('https://',pipeline().parameters.DatabricksWorkspaceID,'.azuredatabricks.net/api/2.0/pipelines/', pipeline().parameters.DLTID,'/updates')", | |
| "type": "Expression" | |
| }, | |
| "connectVia": { | |
| "referenceName": "AutoResolveIntegrationRuntime", | |
| "type": "IntegrationRuntimeReference" | |
| }, | |
| "method": "POST", | |
| "body": { | |
| "value": "@concat('{}')", | |
| "type": "Expression" | |
| }, | |
| "authentication": { | |
| "type": "MSI", | |
| "resource": "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d" | |
| } | |
| } | |
| }, | |
| { | |
| "name": "Wait Until Job Completes", | |
| "type": "Until", | |
| "dependsOn": [ | |
| { | |
| "activity": "Execute DLT Pipeline", | |
| "dependencyConditions": [ | |
| "Succeeded" | |
| ] | |
| } | |
| ], | |
| "userProperties": [], | |
| "typeProperties": { | |
| "expression": { | |
| "value": "@not(equals(variables('JobStatus'),'Running'))", | |
| "type": "Expression" | |
| }, | |
| "activities": [ | |
| { | |
| "name": "Check DLT Update API", | |
| "type": "WebActivity", | |
| "dependsOn": [], | |
| "policy": { | |
| "timeout": "7.00:00:00", | |
| "retry": 0, | |
| "retryIntervalInSeconds": 30, | |
| "secureOutput": false, | |
| "secureInput": false | |
| }, | |
| "userProperties": [], | |
| "typeProperties": { | |
| "url": { | |
| "value": "@concat('https://',pipeline().parameters.DatabricksWorkspaceID,'.azuredatabricks.net/api/2.0/pipelines/', pipeline().parameters.DLTID,'/updates/',activity('Execute DLT Pipeline').output.update_id)", | |
| "type": "Expression" | |
| }, | |
| "connectVia": { | |
| "referenceName": "AutoResolveIntegrationRuntime", | |
| "type": "IntegrationRuntimeReference" | |
| }, | |
| "method": "GET", | |
| "body": { | |
| "job_id": 3895 | |
| }, | |
| "authentication": { | |
| "type": "MSI", | |
| "resource": "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d" | |
| } | |
| } | |
| }, | |
| { | |
| "name": "Set Job Status", | |
| "type": "SetVariable", | |
| "dependsOn": [ | |
| { | |
| "activity": "Check DLT Update API", | |
| "dependencyConditions": [ | |
| "Succeeded" | |
| ] | |
| } | |
| ], | |
| "userProperties": [], | |
| "typeProperties": { | |
| "variableName": "JobStatus", | |
| "value": { | |
| "value": "@if(\nor(\nequals(activity('Check DLT Update API').output.update.state, 'CREATED'),\nor(\nequals(activity('Check DLT Update API').output.update.state, 'WAITING_FOR_RESOURCES'),\nor(\nequals(activity('Check DLT Update API').output.update.state, 'INITIALIZING'),\nor(\nequals(activity('Check DLT Update API').output.update.state, 'SETTING_UP_TABLES'), \nequals(activity('Check DLT Update API').output.update.state, 'RUNNING')\n)\n)\n)\n), \n'Running',\nactivity('Check DLT Update API').output.update.state\n)", | |
| "type": "Expression" | |
| } | |
| } | |
| }, | |
| { | |
| "name": "Wait to Recheck API", | |
| "type": "Wait", | |
| "dependsOn": [ | |
| { | |
| "activity": "Set Job Status", | |
| "dependencyConditions": [ | |
| "Succeeded" | |
| ] | |
| } | |
| ], | |
| "userProperties": [], | |
| "typeProperties": { | |
| "waitTimeInSeconds": { | |
| "value": "@pipeline().parameters.WaitSeconds", | |
| "type": "Expression" | |
| } | |
| } | |
| } | |
| ], | |
| "timeout": "7.00:00:00" | |
| } | |
| } | |
| ], | |
| "parameters": { | |
| "DLTID": { | |
| "type": "string", | |
| "defaultValue": "ab387469-676c-402c-881d-0ba453b4b708" | |
| }, | |
| "DatabricksWorkspaceID": { | |
| "type": "string", | |
| "defaultValue": "adb-5343834423590926.6" | |
| }, | |
| "WaitSeconds": { | |
| "type": "int", | |
| "defaultValue": 30 | |
| }, | |
| "ADFTestValue": { | |
| "type": "string", | |
| "defaultValue": "TestValueADF2" | |
| } | |
| }, | |
| "variables": { | |
| "JobStatus": { | |
| "type": "String", | |
| "defaultValue": "Running" | |
| } | |
| }, | |
| "folder": { | |
| "name": "DLT API" | |
| }, | |
| "annotations": [], | |
| "lastPublishTime": "2021-10-26T17:26:15Z" | |
| }, | |
| "type": "Microsoft.DataFactory/factories/pipelines" | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment