This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def deep_ls(path: str, max_depth=1): | |
""" | |
List all files and folders in specified path and | |
subfolders within maximum recursion depth. | |
""" | |
# List all files in path and apply sorting rules | |
li = mssparkutils.fs.ls(path) | |
# Return all files |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Reusable Functions | |
def azuread_auth(tenant_id: str, client_id: str, client_secret: str, resource_url: str): | |
""" | |
Authenticates Service Principal to the provided Resource URL, and returns the OAuth Access Token | |
""" | |
url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/token" | |
payload= f'grant_type=client_credentials&client_id={client_id}&client_secret={client_secret}&resource={resource_url}' | |
headers = { | |
'Content-Type': 'application/x-www-form-urlencoded' | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Import widgets for Event Hub (i.e. topic) name | |
# Set default | |
dbutils.widgets.text("topic", "your--topic--name") | |
topic = dbutils.widgets.get("topic") | |
# Import required modules | |
from pyspark.sql import * | |
from pyspark.sql.functions import * | |
from pyspark.sql.types import * |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import requests | |
import json | |
import jmespath | |
import pandas as pd | |
from pprint import pprint | |
def azuread_auth(tenant_id: str, client_id: str, client_secret: str, resource_url: str): | |
""" | |
Authenticates Service Principal to the provided Resource URL, and returns the OAuth Access Token |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
### Azure Data Factory info | |
$SubscriptionName = "Azure Subscription Name" | |
$ResourceGroupName = "Resource Group Name containing Data Factory" | |
$DataFactoryName = "Data Factory Name" | |
$AzureSSISName = "SSIS IR Name that will appear in Portal" | |
$AzureSSISDescription = "E.g. Testing SSIS IR Deployment through PowerShell" | |
$AzureSSISLocation = "CanadaCentral" | |
$AzureSSISNodeSize = "Standard_D8_v3" | |
$AzureSSISNodeNumber = 4 | |
$AzureSSISEdition = "Standard" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
param( | |
[string] | |
$gatewayKey | |
) | |
# init log setting | |
$logPath = "$PWD\tracelog.log" | |
"Start to excute SHIRInstall.ps1. `n" | Out-File $logPath | |
function Now-Value() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import requests | |
import json | |
from termcolor import colored | |
def aadtoken(client_id, client_secret, client_tenant): | |
url = "https://login.microsoftonline.com/{}/oauth2/token".format(client_tenant) | |
payload='grant_type=client_credentials&client_id={}&client_secret={}&resource=https%3A%2F%2Fpurview.azure.net'.format(client_id, client_secret) | |
response = requests.request("POST", url, data=payload) | |
return json.loads(response.text)['access_token'] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import requests | |
import json | |
from termcolor import colored | |
def aadtoken(client_id, client_secret, client_tenant): | |
url = "https://login.microsoftonline.com/{}/oauth2/token".format(client_tenant) | |
payload='grant_type=client_credentials&client_id={}&client_secret={}&resource=https%3A%2F%2Fpurview.azure.net'.format(client_id, client_secret) | |
response = requests.request("POST", url, data=payload) | |
return json.loads(response.text)['access_token'] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"entities": [ | |
{ | |
"attributes": { | |
"name": "aemigration.database.windows.net", | |
"qualifiedName": "mssql://aemigration.database.windows.net" | |
}, | |
"collectionId": "aia-purview-new", | |
"status": "ACTIVE", | |
"typeName": "azure_sql_server" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
network: | |
ethernets: | |
eth0: | |
dhcp4: true | |
dhcp4-overrides: | |
route-metric: 100 | |
dhcp6: false | |
addresses: [172.20.0.0/21] | |
nameservers: | |
addresses: [172.20.0.4, 168.63.129.16] |
OlderNewer