The following JSON defines an ADF pipeline that executes an Azure Databricks Job

Execute Databricks Job using MI

{
    "name": "Execute Databricks Job using MI",
    "properties": {
        "description": "Databricks Application ID: 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d",
        "activities": [
            {
                "name": "Execute Jobs API",
                "type": "WebActivity",
                "dependsOn": [],
                "policy": {
                    "timeout": "7.00:00:00",
                    "retry": 0,
                    "retryIntervalInSeconds": 30,
                    "secureOutput": false,
                    "secureInput": false
                },
                "userProperties": [],
                "typeProperties": {
                    "url": {
                        "value": "@concat('https://',pipeline().parameters.DatabricksWorkspaceID,'.azuredatabricks.net/api/2.1/jobs/run-now')",
                        "type": "Expression"
                    },
                    "connectVia": {
                        "referenceName": "AutoResolveIntegrationRuntime",
                        "type": "IntegrationRuntimeReference"
                    },
                    "method": "POST",
                    "body": {
                        "value": "@concat('{\"job_id\":',pipeline().parameters.JobID,'}')",
                        "type": "Expression"
                    },
                    "authentication": {
                        "type": "MSI",
                        "resource": "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d"
                    }
                }
            },
            {
                "name": "Wait Until Job Completes",
                "type": "Until",
                "dependsOn": [
                    {
                        "activity": "Execute Jobs API",
                        "dependencyConditions": [
                            "Succeeded"
                        ]
                    }
                ],
                "userProperties": [],
                "typeProperties": {
                    "expression": {
                        "value": "@not(equals(variables('JobStatus'),'Running'))",
                        "type": "Expression"
                    },
                    "activities": [
                        {
                            "name": "Check Job Run API",
                            "type": "WebActivity",
                            "dependsOn": [],
                            "policy": {
                                "timeout": "7.00:00:00",
                                "retry": 0,
                                "retryIntervalInSeconds": 30,
                                "secureOutput": false,
                                "secureInput": false
                            },
                            "userProperties": [],
                            "typeProperties": {
                                "url": {
                                    "value": "@concat('https://',pipeline().parameters.DatabricksWorkspaceID,'.azuredatabricks.net/api/2.1/jobs/runs/get?run_id=',activity('Execute Jobs API').output.run_id)",
                                    "type": "Expression"
                                },
                                "connectVia": {
                                    "referenceName": "AutoResolveIntegrationRuntime",
                                    "type": "IntegrationRuntimeReference"
                                },
                                "method": "GET",
                                "body": {
                                    "job_id": 3895
                                },
                                "authentication": {
                                    "type": "MSI",
                                    "resource": "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d"
                                }
                            }
                        },
                        {
                            "name": "Set Job Status",
                            "type": "SetVariable",
                            "dependsOn": [
                                {
                                    "activity": "Check Job Run API",
                                    "dependencyConditions": [
                                        "Succeeded"
                                    ]
                                }
                            ],
                            "userProperties": [],
                            "typeProperties": {
                                "variableName": "JobStatus",
                                "value": {
                                    "value": "@if(\nor(\nequals(activity('Check Job Run API').output.state.life_cycle_state, 'PENDING'), equals(activity('Check Job Run API').output.state.life_cycle_state, 'RUNNING')\n), \n'Running',\nactivity('Check Job Run API').output.state.result_state\n)",
                                    "type": "Expression"
                                }
                            }
                        },
                        {
                            "name": "Wait to Recheck API",
                            "type": "Wait",
                            "dependsOn": [
                                {
                                    "activity": "Set Job Status",
                                    "dependencyConditions": [
                                        "Succeeded"
                                    ]
                                }
                            ],
                            "userProperties": [],
                            "typeProperties": {
                                "waitTimeInSeconds": {
                                    "value": "@pipeline().parameters.WaitSeconds",
                                    "type": "Expression"
                                }
                            }
                        }
                    ],
                    "timeout": "7.00:00:00"
                }
            }
        ],
        "parameters": {
            "JobID": {
                "type": "string",
                "defaultValue": "4548"
            },
            "DatabricksWorkspaceID": {
                "type": "string",
                "defaultValue": "adb-5343834423590926.6"
            },
            "WaitSeconds": {
                "type": "int",
                "defaultValue": 30
            }
        },
        "variables": {
            "JobStatus": {
                "type": "String",
                "defaultValue": "Running"
            }
        },
        "folder": {
            "name": "Jobs API"
        },
        "annotations": [],
        "lastPublishTime": "2021-10-26T17:26:15Z"
    },
    "type": "Microsoft.DataFactory/factories/pipelines"
}