BimlScript that is used with an Excel spreadsheet to define datasets and copy pipelines in Azure Data Factory for SQL to ADLS. Needed: List of tables with schema, frequency, indicator of incremental or full load, and field used for window if incremental load
<#@ property name="schema" type="string" #> | |
<#@ property name="table" type="string" #> | |
<#@ property name="frequency" type = "string" #> | |
{ | |
"$schema": "http://datafactories.schema.management.azure.com/schemas/2015-09-01/Microsoft.DataFactory.Table.json", | |
"name": "DS_OnPremSQL_MySourceDB_<#=schema#>_<#=table#>", | |
"properties": { | |
"type": "SqlServerTable", | |
"linkedServiceName": "LS_OnPremSQL_MySourceDB", | |
"structure": [], | |
"external": true, | |
"typeProperties": { | |
"tableName": "<#=schema#>.<#=table#>" | |
}, | |
"availability": { | |
<# if (frequency == "Daily") {#> | |
"frequency": "Day", | |
<#} else if (frequency == "Hourly") {#> | |
"frequency": "Hour", | |
"style": "EndOfInterval", | |
<# } #> | |
"interval": 1 | |
} | |
} | |
} |
<#@ property name="schema" type="string" #> | |
<#@ property name="table" type="string" #> | |
<#@ property name="frequency" type = "string" #> | |
<#@ property name="scope" type="string" #> | |
{ | |
"$schema": "http://datafactories.schema.management.azure.com/schemas/2015-09-01/Microsoft.DataFactory.Table.json", | |
"name": "DS_DataLake_ClawfootSupply_<#=schema#>_<#=table#>", | |
"properties": { | |
"type": "AzureDataLakeStore", | |
"linkedServiceName": "LS_ADLS_SHDataLake", | |
"structure": [ ], | |
"typeProperties": { | |
"folderPath": "/raw/clawfootsupply/<#=schema#>_<#=table#>/<#=scope.ToLower()#>/{Year}/{Month}/{Day}/", | |
"fileName": "<#=schema#>_<#=table#>_{Hour}{Minute}{Second}.txt", | |
"format": { | |
"type": "TextFormat", | |
"columnDelimiter": "|", | |
"quoteChar": "\"", | |
"nullValue": "" | |
}, | |
"partitionedBy": [ | |
{ | |
"name": "Year", | |
"value": { | |
"type": "DateTime", | |
"date": "SliceStart", | |
"format": "yyyy" | |
} | |
}, | |
{ | |
"name": "Month", | |
"value": { | |
"type": "DateTime", | |
"date": "SliceStart", | |
"format": "%M" | |
} | |
}, | |
{ | |
"name": "Day", | |
"value": { | |
"type": "DateTime", | |
"date": "SliceStart", | |
"format": "%d" | |
} | |
}, | |
{ | |
"name": "Hour", | |
"value": { | |
"type": "DateTime", | |
"date": "SliceStart", | |
"format": "hh" | |
} | |
}, | |
{ | |
"name": "Minute", | |
"value": { | |
"type": "DateTime", | |
"date": "SliceStart", | |
"format": "mm" | |
} | |
}, | |
{ | |
"name": "Second", | |
"value": { | |
"type": "DateTime", | |
"date": "SliceStart", | |
"format": "ss" | |
} | |
} | |
] | |
}, | |
"availability": { | |
<# if (frequency == "Daily") {#> | |
"frequency": "Day", | |
<#} else if (frequency == "Hourly") {#> | |
"frequency": "Hour", | |
"style": "EndOfInterval", | |
<# } #> | |
"interval": 1 | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment