Skip to content

Instantly share code, notes, and snippets.

@flanakin

flanakin/README.md

Last active March 15, 2024 00:07
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save flanakin/419b5ed778d12a109b297d0dd406c10f to your computer and use it in GitHub Desktop.
Save flanakin/419b5ed778d12a109b297d0dd406c10f to your computer and use it in GitHub Desktop.
Download ZIP
FinOps hubs 0.2.1-rc3
Raw
README.md

This is a release candidate for FinOps hubs 0.2.1 to resolve 2 known bugs where ADF shows a syntax error and the main processing trigger was not being started.

Deploy To Azure   Deploy To Azure US Gov   Deploy To Azure China

To upgrade, make sure you create new FOCUS exports and use the new Power BI reports from the 0.2 release. You will need to run exports for past dates in order to see historical data. Amortized exports from previous releases will not be moved over and older Power BI reports will not work since the schema has changed.

Known issues

No known issues. Please create an issue in the FinOps toolkit if you discover anything.

Raw
azuredeploy.json
{
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.24.24.22086",
"templateHash": "6371350577264419703"
}
},
"parameters": {
"hubName": {
"type": "string",
"metadata": {
"description": "Optional. Name of the hub. Used to ensure unique resource names. Default: \"finops-hub\"."
}
},
"location": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": "Optional. Azure location where all resources should be created. See https://aka.ms/azureregions. Default: Same as deployment."
}
},
"storageSku": {
"type": "string",
"defaultValue": "Premium_LRS",
"allowedValues": [
"Premium_LRS",
"Premium_ZRS"
],
"metadata": {
"description": "Optional. Storage SKU to use. LRS = Lowest cost, ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Allowed: Premium_LRS, Premium_ZRS. Default: Premium_LRS."
}
},
"tags": {
"type": "object",
"defaultValue": {},
"metadata": {
"description": "Optional. Tags to apply to all resources. We will also add the cm-resource-parent tag for improved cost roll-ups in Cost Management."
}
},
"tagsByResource": {
"type": "object",
"defaultValue": {},
"metadata": {
"description": "Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources."
}
},
"exportScopes": {
"type": "array",
"defaultValue": [],
"metadata": {
"description": "Optional. List of scope IDs to create exports for."
}
}
},
"resources": [
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2022-09-01",
"name": "hub",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"hubName": {
"value": "[parameters('hubName')]"
},
"location": {
"value": "[parameters('location')]"
},
"storageSku": {
"value": "[parameters('storageSku')]"
},
"tags": {
"value": "[parameters('tags')]"
},
"tagsByResource": {
"value": "[parameters('tagsByResource')]"
},
"exportScopes": {
"value": "[parameters('exportScopes')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.24.24.22086",
"templateHash": "2610829918662778812"
}
},
"parameters": {
"hubName": {
"type": "string",
"metadata": {
"description": "Optional. Name of the hub. Used to ensure unique resource names. Default: \"finops-hub\"."
}
},
"location": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": "Optional. Azure location where all resources should be created. See https://aka.ms/azureregions. Default: (resource group location)."
}
},
"storageSku": {
"type": "string",
"defaultValue": "Premium_LRS",
"allowedValues": [
"Premium_LRS",
"Premium_ZRS"
],
"metadata": {
"description": "Optional. Storage SKU to use. LRS = Lowest cost, ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Allowed: Premium_LRS, Premium_ZRS. Default: Premium_LRS."
}
},
"tags": {
"type": "object",
"defaultValue": {},
"metadata": {
"description": "Optional. Tags to apply to all resources. We will also add the cm-resource-parent tag for improved cost roll-ups in Cost Management."
}
},
"tagsByResource": {
"type": "object",
"defaultValue": {},
"metadata": {
"description": "Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources."
}
},
"exportScopes": {
"type": "array",
"metadata": {
"description": "Optional. List of scope IDs to create exports for."
}
},
"convertToParquet": {
"type": "bool",
"defaultValue": true,
"metadata": {
"description": "Optional. Indicates whether ingested data should be converted to Parquet. Default: true."
}
},
"enableDefaultTelemetry": {
"type": "bool",
"defaultValue": true,
"metadata": {
"description": "Optional. Enable telemetry to track anonymous module usage trends, monitor for bugs, and improve future releases."
}
}
},
"variables": {
"resourceTags": "[union(parameters('tags'), createObject('cm-resource-parent', format('{0}/providers/Microsoft.Cloud/hubs/{1}', resourceGroup().id, parameters('hubName'))))]",
"uniqueSuffix": "[uniqueString(parameters('hubName'), resourceGroup().id)]",
"dataFactoryPrefix": "[format('{0}-engine', replace(parameters('hubName'), '_', '-'))]",
"dataFactorySuffix": "[format('-{0}', variables('uniqueSuffix'))]",
"dataFactoryName": "[replace(format('{0}{1}', take(variables('dataFactoryPrefix'), sub(63, length(variables('dataFactorySuffix')))), variables('dataFactorySuffix')), '--', '-')]",
"telemetryId": "00f120b5-2007-6120-0000-40b000000000",
"finOpsToolkitVersion": "placeholder"
},
"resources": [
{
"condition": "[parameters('enableDefaultTelemetry')]",
"type": "Microsoft.Resources/deployments",
"apiVersion": "2022-09-01",
"name": "[format('pid-{0}-{1}', variables('telemetryId'), uniqueString(deployment().name, parameters('location')))]",
"properties": {
"mode": "Incremental",
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "FinOps toolkit",
"version": "[variables('finOpsToolkitVersion')]"
}
},
"resources": []
}
}
},
{
"type": "Microsoft.DataFactory/factories",
"apiVersion": "2018-06-01",
"name": "[variables('dataFactoryName')]",
"location": "[parameters('location')]",
"tags": "[union(variables('resourceTags'), if(contains(parameters('tagsByResource'), 'Microsoft.DataFactory/factories'), parameters('tagsByResource')['Microsoft.DataFactory/factories'], createObject()))]",
"identity": {
"type": "SystemAssigned"
},
"properties": "[union(createObject(), createObject('globalConfigurations', createObject('PipelineBillingEnabled', 'true')))]"
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2022-09-01",
"name": "storage",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"hubName": {
"value": "[parameters('hubName')]"
},
"uniqueSuffix": {
"value": "[variables('uniqueSuffix')]"
},
"sku": {
"value": "[parameters('storageSku')]"
},
"location": {
"value": "[parameters('location')]"
},
"tags": {
"value": "[variables('resourceTags')]"
},
"tagsByResource": {
"value": "[parameters('tagsByResource')]"
},
"exportScopes": {
"value": "[parameters('exportScopes')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.24.24.22086",
"templateHash": "9609675916139055971"
}
},
"parameters": {
"hubName": {
"type": "string",
"metadata": {
"description": "Required. Name of the hub. Used to ensure unique resource names."
}
},
"uniqueSuffix": {
"type": "string",
"metadata": {
"description": "Required. Suffix to add to the storage account name to ensure uniqueness."
}
},
"location": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": "Optional. Azure location where all resources should be created. See https://aka.ms/azureregions. Default: (resource group location)."
}
},
"sku": {
"type": "string",
"defaultValue": "Premium_LRS",
"allowedValues": [
"Premium_LRS",
"Premium_ZRS"
],
"metadata": {
"description": "Optional. Storage SKU to use. LRS = Lowest cost, ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Allowed: Premium_LRS, Premium_ZRS. Default: Premium_LRS."
}
},
"tags": {
"type": "object",
"defaultValue": {},
"metadata": {
"description": "Optional. Tags to apply to all resources. We will also add the cm-resource-parent tag for improved cost roll-ups in Cost Management."
}
},
"tagsByResource": {
"type": "object",
"defaultValue": {},
"metadata": {
"description": "Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources."
}
},
"exportScopes": {
"type": "array",
"metadata": {
"description": "Optional. List of scope IDs to create exports for."
}
}
},
"variables": {
"$fxv#0": "placeholder",
"$fxv#1": "# Copyright (c) Microsoft Corporation.\r\n# Licensed under the MIT License.\r\n\r\nWrite-Output \"Updating settings.json file...\"\r\nWrite-Output \" Storage account: $env:storageAccountName\"\r\nWrite-Output \" Container: $env:containerName\"\r\n\r\n$validateScopes = { $_.Length -gt 45 }\r\n\r\n# Initialize variables\r\n$fileName = 'settings.json'\r\n$filePath = Join-Path -Path . -ChildPath $fileName\r\n$newScopes = $env:exportScopes.Split('|') | Where-Object $validateScopes | ForEach-Object { @{ scope = $_ } }\r\n\r\n# Get storage context\r\n$storageContext = @{\r\n Context = New-AzStorageContext -StorageAccountName $env:storageAccountName -UseConnectedAccount\r\n Container = $env:containerName\r\n}\r\n\r\n# Download existing settings, if they exist\r\n$blob = Get-AzStorageBlobContent @storageContext -Blob $fileName -Destination $filePath -Force\r\nif ($blob) {\r\n Write-Output \"Existing settings.json file found. Updating...\"\r\n $text = Get-Content $filePath -Raw\r\n Write-Output \"---------\"\r\n Write-Output $text\r\n Write-Output \"---------\"\r\n $json = $text | ConvertFrom-Json\r\n\r\n # Rename exportScopes to scopes + convert to object array\r\n if ($json.exportScopes) {\r\n Write-Output \" Updating exportScopes...\"\r\n if ($json.exportScopes[0] -is [string]) {\r\n Write-Output \" Converting string array to object array...\"\r\n $json.exportScopes = $json.exportScopes | Where-Object $validateScopes | ForEach-Object { @{ scope = $_ } }\r\n if (-not ($json.exportScopes -is [array])) {\r\n Write-Output \" Converting single object to object array...\"\r\n $json.exportScopes = @($json.exportScopes)\r\n }\r\n }\r\n\r\n Write-Output \" Renaming to 'scopes'...\"\r\n $json | Add-Member -MemberType NoteProperty -Name scopes -Value $json.exportScopes\r\n $json.PSObject.Properties.Remove('exportScopes')\r\n }\r\n}\r\n\r\n# Set default if not found\r\nif (!$json) {\r\n Write-Output \"No existing settings.json file found. Creating new file...\"\r\n $json = [ordered]@{\r\n '$schema' = 'https://aka.ms/finops/hubs/settings-schema'\r\n type = 'HubInstance'\r\n version = ''\r\n learnMore = 'https://aka.ms/finops/hubs'\r\n scopes = @()\r\n }\r\n}\r\n\r\n# Updating settings\r\nWrite-Output \"Updating version to $env:ftkVersion...\"\r\n$json.version = $env:ftkVersion\r\nif ($newScopes) {\r\n Write-Output \"Merging $($newScopes.Count) scopes...\"\r\n $json.scopes = Compare-Object -ReferenceObject $json.scopes -DifferenceObject $newScopes -Property scope -PassThru -IncludeEqual\r\n\r\n # Remove the SideIndicator property from the Compare-Object output\r\n $json.scopes | ForEach-Object { $_.PSObject.Properties.Remove('SideIndicator') } | ConvertTo-Json\r\n\r\n if (-not ($json.scopes -is [array])) {\r\n $json.scopes = @($json.scopes)\r\n }\r\n Write-Output \"$($json.scopes.Count) scopes found.\"\r\n}\r\n$text = $json | ConvertTo-Json\r\nWrite-Output \"---------\"\r\nWrite-Output $text\r\nWrite-Output \"---------\"\r\n$text | Out-File $filePath\r\n\r\n# Upload new/updated settings\r\nWrite-Output \"Uploading settings.json file...\"\r\nSet-AzStorageBlobContent @storageContext -File $filePath -Force\r\n",
"safeHubName": "[replace(replace(toLower(parameters('hubName')), '-', ''), '_', '')]",
"storageAccountSuffix": "[parameters('uniqueSuffix')]",
"storageAccountName": "[format('{0}{1}', take(variables('safeHubName'), sub(24, length(variables('storageAccountSuffix')))), variables('storageAccountSuffix'))]",
"blobUploadRbacRoles": [
"ba92f5b4-2d11-453d-a403-e96b0029c9fe",
"e40ec5ca-96e0-45a2-b4ff-59039f2c2b59"
]
},
"resources": [
{
"type": "Microsoft.Storage/storageAccounts",
"apiVersion": "2021-08-01",
"name": "[variables('storageAccountName')]",
"location": "[parameters('location')]",
"sku": {
"name": "[parameters('sku')]"
},
"kind": "BlockBlobStorage",
"tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.Storage/storageAccounts'), parameters('tagsByResource')['Microsoft.Storage/storageAccounts'], createObject()))]",
"properties": {
"supportsHttpsTrafficOnly": true,
"isHnsEnabled": true,
"minimumTlsVersion": "TLS1_2",
"allowBlobPublicAccess": false
}
},
{
"type": "Microsoft.Storage/storageAccounts/blobServices",
"apiVersion": "2021-06-01",
"name": "[format('{0}/{1}', variables('storageAccountName'), 'default')]",
"dependsOn": [
"[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]"
]
},
{
"type": "Microsoft.Storage/storageAccounts/blobServices/containers",
"apiVersion": "2021-06-01",
"name": "[format('{0}/{1}/{2}', variables('storageAccountName'), 'default', 'config')]",
"properties": {
"publicAccess": "None",
"metadata": {}
},
"dependsOn": [
"[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('storageAccountName'), 'default')]"
]
},
{
"type": "Microsoft.Storage/storageAccounts/blobServices/containers",
"apiVersion": "2021-06-01",
"name": "[format('{0}/{1}/{2}', variables('storageAccountName'), 'default', 'msexports')]",
"properties": {
"publicAccess": "None",
"metadata": {}
},
"dependsOn": [
"[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('storageAccountName'), 'default')]"
]
},
{
"type": "Microsoft.Storage/storageAccounts/blobServices/containers",
"apiVersion": "2021-06-01",
"name": "[format('{0}/{1}/{2}', variables('storageAccountName'), 'default', 'ingestion')]",
"properties": {
"publicAccess": "None",
"metadata": {}
},
"dependsOn": [
"[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('storageAccountName'), 'default')]"
]
},
{
"type": "Microsoft.ManagedIdentity/userAssignedIdentities",
"apiVersion": "2023-01-31",
"name": "[format('{0}_blobManager', variables('storageAccountName'))]",
"tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.ManagedIdentity/userAssignedIdentities'), parameters('tagsByResource')['Microsoft.ManagedIdentity/userAssignedIdentities'], createObject()))]",
"location": "[parameters('location')]"
},
{
"copy": {
"name": "identityRoleAssignments",
"count": "[length(variables('blobUploadRbacRoles'))]"
},
"type": "Microsoft.Authorization/roleAssignments",
"apiVersion": "2022-04-01",
"name": "[guid(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), variables('blobUploadRbacRoles')[copyIndex()], resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_blobManager', variables('storageAccountName'))))]",
"properties": {
"roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', variables('blobUploadRbacRoles')[copyIndex()])]",
"principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_blobManager', variables('storageAccountName'))), '2023-01-31').principalId]",
"principalType": "ServicePrincipal"
},
"dependsOn": [
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_blobManager', variables('storageAccountName')))]",
"[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]"
]
},
{
"type": "Microsoft.Resources/deploymentScripts",
"apiVersion": "2020-10-01",
"name": "uploadSettings",
"kind": "AzurePowerShell",
"location": "[if(startsWith(parameters('location'), 'china'), 'chinaeast2', parameters('location'))]",
"tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.Resources/deploymentScripts'), parameters('tagsByResource')['Microsoft.Resources/deploymentScripts'], createObject()))]",
"identity": {
"type": "UserAssigned",
"userAssignedIdentities": {
"[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_blobManager', variables('storageAccountName'))))]": {}
}
},
"properties": {
"azPowerShellVersion": "8.0",
"retentionInterval": "PT1H",
"environmentVariables": [
{
"name": "ftkVersion",
"value": "[variables('$fxv#0')]"
},
{
"name": "exportScopes",
"value": "[join(parameters('exportScopes'), '|')]"
},
{
"name": "storageAccountName",
"value": "[variables('storageAccountName')]"
},
{
"name": "containerName",
"value": "config"
}
],
"scriptContent": "[variables('$fxv#1')]"
},
"dependsOn": [
"[resourceId('Microsoft.Storage/storageAccounts/blobServices/containers', variables('storageAccountName'), 'default', 'config')]",
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_blobManager', variables('storageAccountName')))]",
"identityRoleAssignments"
]
}
],
"outputs": {
"resourceId": {
"type": "string",
"metadata": {
"description": "The resource ID of the storage account."
},
"value": "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]"
},
"name": {
"type": "string",
"metadata": {
"description": "The name of the storage account."
},
"value": "[variables('storageAccountName')]"
},
"configContainer": {
"type": "string",
"metadata": {
"description": "The name of the container used for configuration settings."
},
"value": "config"
},
"exportContainer": {
"type": "string",
"metadata": {
"description": "The name of the container used for Cost Management exports."
},
"value": "msexports"
},
"ingestionContainer": {
"type": "string",
"metadata": {
"description": "The name of the container used for normalized data ingestion."
},
"value": "ingestion"
}
}
}
}
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2022-09-01",
"name": "dataFactoryResources",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"dataFactoryName": {
"value": "[variables('dataFactoryName')]"
},
"convertToParquet": {
"value": "[parameters('convertToParquet')]"
},
"keyVaultName": {
"value": "[reference(resourceId('Microsoft.Resources/deployments', 'keyVault'), '2022-09-01').outputs.name.value]"
},
"storageAccountName": {
"value": "[reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.name.value]"
},
"exportContainerName": {
"value": "[reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.exportContainer.value]"
},
"ingestionContainerName": {
"value": "[reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.ingestionContainer.value]"
},
"location": {
"value": "[parameters('location')]"
},
"tags": {
"value": "[variables('resourceTags')]"
},
"tagsByResource": {
"value": "[parameters('tagsByResource')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.24.24.22086",
"templateHash": "5738194981634133446"
}
},
"parameters": {
"dataFactoryName": {
"type": "string",
"metadata": {
"description": "Optional. Name of the hub. Used to ensure unique resource names. Default: \"finops-hub\"."
}
},
"keyVaultName": {
"type": "string",
"metadata": {
"description": "Required. The name of the Azure Key Vault instance."
}
},
"storageAccountName": {
"type": "string",
"metadata": {
"description": "Required. The name of the Azure storage account instance."
}
},
"exportContainerName": {
"type": "string",
"metadata": {
"description": "Required. The name of the container where Cost Management data is exported."
}
},
"ingestionContainerName": {
"type": "string",
"metadata": {
"description": "Required. The name of the container where normalized data is ingested."
}
},
"convertToParquet": {
"type": "bool",
"defaultValue": true,
"metadata": {
"description": "Optional. Indicates whether ingested data should be converted to Parquet. Default: true."
}
},
"location": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": "Optional. The location to use for the managed identity and deployment script to auto-start triggers. Default = (resource group location)."
}
},
"tags": {
"type": "object",
"defaultValue": {},
"metadata": {
"description": "Optional. Tags to apply to all resources. We will also add the cm-resource-parent tag for improved cost roll-ups in Cost Management."
}
},
"tagsByResource": {
"type": "object",
"defaultValue": {},
"metadata": {
"description": "Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources."
}
}
},
"variables": {
"copy": [
{
"name": "focusCostMappings",
"count": "[length(range(0, length(variables('focusCostColumns'))))]",
"input": {
"source": {
"name": "[variables('focusCostColumns')[range(0, length(variables('focusCostColumns')))[copyIndex('focusCostMappings')]].name]",
"type": "[variables('focusCostColumns')[range(0, length(variables('focusCostColumns')))[copyIndex('focusCostMappings')]].type]"
},
"sink": {
"name": "[variables('focusCostColumns')[range(0, length(variables('focusCostColumns')))[copyIndex('focusCostMappings')]].name]"
}
}
}
],
"$fxv#0": "# Copyright (c) Microsoft Corporation.\r\n# Licensed under the MIT License.\r\n\r\n# Init outputs\r\n$DeploymentScriptOutputs = @{}\r\n\r\n# \r\n$adfParams = @{\r\n ResourceGroupName = $env:DataFactoryResourceGroup\r\n DataFactoryName = $env:DataFactoryName\r\n}\r\n\r\n# Delete old triggers\r\n$triggers = Get-AzDataFactoryV2Trigger @adfParams -ErrorAction SilentlyContinue `\r\n| Where-Object { $_.Name -match '^msexports?$' }\r\n$DeploymentScriptOutputs[\"stopTriggers\"] = $triggers | Stop-AzDataFactoryV2Trigger -Force -ErrorAction SilentlyContinue\r\n$DeploymentScriptOutputs[\"deleteTriggers\"] = $triggers | Remove-AzDataFactoryV2Trigger -Force -ErrorAction SilentlyContinue\r\n\r\n# Delete old pipelines\r\n$DeploymentScriptOutputs[\"pipelines\"] = Get-AzDataFactoryV2Pipeline @adfParams -ErrorAction SilentlyContinue `\r\n| Where-Object { $_.Name -match '^msexports_(extract|transform)$' } `\r\n| Remove-AzDataFactoryV2Pipeline -Force -ErrorAction SilentlyContinue",
"$fxv#1": "# Copyright (c) Microsoft Corporation.\r\n# Licensed under the MIT License.\r\n\r\nParam(\r\n [switch] $Stop\r\n)\r\n\r\n# Init outputs\r\n$DeploymentScriptOutputs = @{}\r\n\r\nif (-not $Stop) {\r\n Start-Sleep -Seconds 10\r\n}\r\n\r\n# Loop thru triggers\r\n$env:Triggers.Split('|') `\r\n| ForEach-Object {\r\n $trigger = $_\r\n if ($Stop) {\r\n Write-Host \"Stopping trigger $trigger...\" -NoNewline\r\n $triggerOutput = Stop-AzDataFactoryV2Trigger `\r\n -ResourceGroupName $env:DataFactoryResourceGroup `\r\n -DataFactoryName $env:DataFactoryName `\r\n -Name $trigger `\r\n -Force `\r\n -ErrorAction SilentlyContinue # Ignore errors, since the trigger may not exist\r\n } else {\r\n Write-Host \"Starting trigger $trigger...\" -NoNewline\r\n $triggerOutput = Start-AzDataFactoryV2Trigger `\r\n -ResourceGroupName $env:DataFactoryResourceGroup `\r\n -DataFactoryName $env:DataFactoryName `\r\n -Name $trigger `\r\n -Force\r\n }\r\n if ($triggerOutput) {\r\n Write-Host 'done'\r\n } else {\r\n Write-Host 'failed'\r\n }\r\n $DeploymentScriptOutputs[$trigger] = $triggerOutput\r\n}\r\n\r\nif ($Stop) {\r\n Start-Sleep -Seconds 10\r\n}\r\n",
"$fxv#2": "# Copyright (c) Microsoft Corporation.\r\n# Licensed under the MIT License.\r\n\r\nParam(\r\n [switch] $Stop\r\n)\r\n\r\n# Init outputs\r\n$DeploymentScriptOutputs = @{}\r\n\r\nif (-not $Stop) {\r\n Start-Sleep -Seconds 10\r\n}\r\n\r\n# Loop thru triggers\r\n$env:Triggers.Split('|') `\r\n| ForEach-Object {\r\n $trigger = $_\r\n if ($Stop) {\r\n Write-Host \"Stopping trigger $trigger...\" -NoNewline\r\n $triggerOutput = Stop-AzDataFactoryV2Trigger `\r\n -ResourceGroupName $env:DataFactoryResourceGroup `\r\n -DataFactoryName $env:DataFactoryName `\r\n -Name $trigger `\r\n -Force `\r\n -ErrorAction SilentlyContinue # Ignore errors, since the trigger may not exist\r\n } else {\r\n Write-Host \"Starting trigger $trigger...\" -NoNewline\r\n $triggerOutput = Start-AzDataFactoryV2Trigger `\r\n -ResourceGroupName $env:DataFactoryResourceGroup `\r\n -DataFactoryName $env:DataFactoryName `\r\n -Name $trigger `\r\n -Force\r\n }\r\n if ($triggerOutput) {\r\n Write-Host 'done'\r\n } else {\r\n Write-Host 'failed'\r\n }\r\n $DeploymentScriptOutputs[$trigger] = $triggerOutput\r\n}\r\n\r\nif ($Stop) {\r\n Start-Sleep -Seconds 10\r\n}\r\n",
"datasetPropsDelimitedText": {
"columnDelimiter": ",",
"compressionLevel": "Optimal",
"escapeChar": "\"",
"firstRowAsHeader": true,
"quoteChar": "\""
},
"datasetPropsCommon": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@{dataset().fileName}",
"type": "Expression"
},
"folderPath": {
"value": "@{dataset().folderName}",
"type": "Expression"
}
}
},
"safeExportContainerName": "[replace(format('{0}', parameters('exportContainerName')), '-', '_')]",
"safeIngestionContainerName": "[replace(format('{0}', parameters('ingestionContainerName')), '-', '_')]",
"exportFileAddedTriggerName": "[format('{0}_FileAdded', variables('safeExportContainerName'))]",
"allHubTriggers": [
"[variables('exportFileAddedTriggerName')]"
],
"autoStartRbacRoles": [
"673868aa-7521-48a0-acc6-0f60742d39f5",
"e40ec5ca-96e0-45a2-b4ff-59039f2c2b59"
],
"focusCostColumns": [
{
"name": "AvailabilityZone",
"type": "String"
},
{
"name": "BilledCost",
"type": "Decimal"
},
{
"name": "BillingAccountId",
"type": "String"
},
{
"name": "BillingAccountName",
"type": "String"
},
{
"name": "BillingAccountType",
"type": "String"
},
{
"name": "BillingCurrency",
"type": "String"
},
{
"name": "BillingPeriodEnd",
"type": "DateTime"
},
{
"name": "BillingPeriodStart",
"type": "DateTime"
},
{
"name": "ChargeCategory",
"type": "String"
},
{
"name": "ChargeDescription",
"type": "String"
},
{
"name": "ChargeFrequency",
"type": "String"
},
{
"name": "ChargePeriodEnd",
"type": "DateTime"
},
{
"name": "ChargePeriodStart",
"type": "DateTime"
},
{
"name": "ChargeSubcategory",
"type": "String"
},
{
"name": "CommitmentDiscountCategory",
"type": "String"
},
{
"name": "CommitmentDiscountId",
"type": "String"
},
{
"name": "CommitmentDiscountName",
"type": "String"
},
{
"name": "CommitmentDiscountType",
"type": "String"
},
{
"name": "EffectiveCost",
"type": "Decimal"
},
{
"name": "InvoiceIssuerName",
"type": "String"
},
{
"name": "ListCost",
"type": "Decimal"
},
{
"name": "ListUnitPrice",
"type": "Decimal"
},
{
"name": "PricingCategory",
"type": "String"
},
{
"name": "PricingQuantity",
"type": "Decimal"
},
{
"name": "PricingUnit",
"type": "String"
},
{
"name": "ProviderName",
"type": "String"
},
{
"name": "PublisherName",
"type": "String"
},
{
"name": "Region",
"type": "String"
},
{
"name": "ResourceId",
"type": "String"
},
{
"name": "ResourceName",
"type": "String"
},
{
"name": "ResourceType",
"type": "String"
},
{
"name": "ServiceCategory",
"type": "String"
},
{
"name": "ServiceName",
"type": "String"
},
{
"name": "SkuId",
"type": "String"
},
{
"name": "SkuPriceId",
"type": "String"
},
{
"name": "SubAccountId",
"type": "String"
},
{
"name": "SubAccountName",
"type": "String"
},
{
"name": "SubAccountType",
"type": "String"
},
{
"name": "Tags",
"type": "String"
},
{
"name": "UsageQuantity",
"type": "Decimal"
},
{
"name": "UsageUnit",
"type": "String"
},
{
"name": "x_AccountName",
"type": "String"
},
{
"name": "x_AccountOwnerId",
"type": "String"
},
{
"name": "x_BilledCostInUsd",
"type": "Decimal"
},
{
"name": "x_BilledUnitPrice",
"type": "Decimal"
},
{
"name": "x_BillingAccountId",
"type": "String"
},
{
"name": "x_BillingAccountName",
"type": "String"
},
{
"name": "x_BillingExchangeRate",
"type": "Decimal"
},
{
"name": "x_BillingExchangeRateDate",
"type": "DateTime"
},
{
"name": "x_BillingProfileId",
"type": "String"
},
{
"name": "x_BillingProfileName",
"type": "String"
},
{
"name": "x_ChargeId",
"type": "String"
},
{
"name": "x_CostAllocationRuleName",
"type": "String"
},
{
"name": "x_CostCenter",
"type": "String"
},
{
"name": "x_CustomerId",
"type": "String"
},
{
"name": "x_CustomerName",
"type": "String"
},
{
"name": "x_EffectiveCostInUsd",
"type": "Decimal"
},
{
"name": "x_EffectiveUnitPrice",
"type": "Decimal"
},
{
"name": "x_InvoiceId",
"type": "String"
},
{
"name": "x_InvoiceIssuerId",
"type": "String"
},
{
"name": "x_InvoiceSectionId",
"type": "String"
},
{
"name": "x_InvoiceSectionName",
"type": "String"
},
{
"name": "x_OnDemandCost",
"type": "Decimal"
},
{
"name": "x_OnDemandCostInUsd",
"type": "Decimal"
},
{
"name": "x_OnDemandUnitPrice",
"type": "Decimal"
},
{
"name": "x_PartnerCreditApplied",
"type": "Boolean"
},
{
"name": "x_PartnerCreditRate",
"type": "Decimal"
},
{
"name": "x_PricingBlockSize",
"type": "Decimal"
},
{
"name": "x_PricingCurrency",
"type": "String"
},
{
"name": "x_PricingSubcategory",
"type": "String"
},
{
"name": "x_PricingUnitDescription",
"type": "String"
},
{
"name": "x_PublisherCategory",
"type": "String"
},
{
"name": "x_PublisherId",
"type": "String"
},
{
"name": "x_ResellerId",
"type": "String"
},
{
"name": "x_ResellerName",
"type": "String"
},
{
"name": "x_ResourceGroupName",
"type": "String"
},
{
"name": "x_ResourceType",
"type": "String"
},
{
"name": "x_ServicePeriodEnd",
"type": "DateTime"
},
{
"name": "x_ServicePeriodStart",
"type": "DateTime"
},
{
"name": "x_SkuDescription",
"type": "String"
},
{
"name": "x_SkuDetails",
"type": "String"
},
{
"name": "x_SkuIsCreditEligible",
"type": "Boolean"
},
{
"name": "x_SkuMeterCategory",
"type": "String"
},
{
"name": "x_SkuMeterId",
"type": "String"
},
{
"name": "x_SkuMeterName",
"type": "String"
},
{
"name": "x_SkuMeterSubcategory",
"type": "String"
},
{
"name": "x_SkuOfferId",
"type": "String"
},
{
"name": "x_SkuOrderId",
"type": "String"
},
{
"name": "x_SkuOrderName",
"type": "String"
},
{
"name": "x_SkuPartNumber",
"type": "String"
},
{
"name": "x_SkuRegion",
"type": "String"
},
{
"name": "x_SkuServiceFamily",
"type": "String"
},
{
"name": "x_SkuTerm",
"type": "String"
},
{
"name": "x_SkuTier",
"type": "String"
}
]
},
"resources": [
{
"type": "Microsoft.Resources/deploymentScripts",
"apiVersion": "2020-10-01",
"name": "[format('{0}_deleteOldResources', parameters('dataFactoryName'))]",
"location": "[parameters('location')]",
"identity": {
"type": "UserAssigned",
"userAssignedIdentities": {
"[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName'))))]": {}
}
},
"kind": "AzurePowerShell",
"tags": "[parameters('tags')]",
"properties": {
"azPowerShellVersion": "8.0",
"retentionInterval": "PT1H",
"cleanupPreference": "OnSuccess",
"scriptContent": "[variables('$fxv#0')]",
"environmentVariables": [
{
"name": "DataFactorySubscriptionId",
"value": "[subscription().id]"
},
{
"name": "DataFactoryResourceGroup",
"value": "[resourceGroup().name]"
},
{
"name": "DataFactoryName",
"value": "[parameters('dataFactoryName')]"
}
]
},
"dependsOn": [
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName')))]",
"identityRoleAssignments"
]
},
{
"type": "Microsoft.ManagedIdentity/userAssignedIdentities",
"apiVersion": "2023-01-31",
"name": "[format('{0}_triggerManager', parameters('dataFactoryName'))]",
"location": "[parameters('location')]",
"tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.ManagedIdentity/userAssignedIdentities'), parameters('tagsByResource')['Microsoft.ManagedIdentity/userAssignedIdentities'], createObject()))]"
},
{
"copy": {
"name": "identityRoleAssignments",
"count": "[length(variables('autoStartRbacRoles'))]"
},
"type": "Microsoft.Authorization/roleAssignments",
"apiVersion": "2022-04-01",
"scope": "[format('Microsoft.DataFactory/factories/{0}', parameters('dataFactoryName'))]",
"name": "[guid(resourceId('Microsoft.DataFactory/factories', parameters('dataFactoryName')), variables('autoStartRbacRoles')[copyIndex()], resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName'))))]",
"properties": {
"roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', variables('autoStartRbacRoles')[copyIndex()])]",
"principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName'))), '2023-01-31').principalId]",
"principalType": "ServicePrincipal"
},
"dependsOn": [
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName')))]"
]
},
{
"type": "Microsoft.Resources/deploymentScripts",
"apiVersion": "2020-10-01",
"name": "[format('{0}_stopHubTriggers', parameters('dataFactoryName'))]",
"location": "[if(startsWith(parameters('location'), 'china'), 'chinaeast2', parameters('location'))]",
"identity": {
"type": "UserAssigned",
"userAssignedIdentities": {
"[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName'))))]": {}
}
},
"kind": "AzurePowerShell",
"tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.Resources/deploymentScripts'), parameters('tagsByResource')['Microsoft.Resources/deploymentScripts'], createObject()))]",
"properties": {
"azPowerShellVersion": "8.0",
"retentionInterval": "PT1H",
"cleanupPreference": "OnSuccess",
"scriptContent": "[variables('$fxv#1')]",
"arguments": "-Stop",
"environmentVariables": [
{
"name": "DataFactorySubscriptionId",
"value": "[subscription().id]"
},
{
"name": "DataFactoryResourceGroup",
"value": "[resourceGroup().name]"
},
{
"name": "DataFactoryName",
"value": "[parameters('dataFactoryName')]"
},
{
"name": "Triggers",
"value": "[join(variables('allHubTriggers'), '|')]"
}
]
},
"dependsOn": [
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName')))]",
"identityRoleAssignments"
]
},
{
"type": "Microsoft.DataFactory/factories/linkedservices",
"apiVersion": "2018-06-01",
"name": "[format('{0}/{1}', parameters('dataFactoryName'), 'keyVault')]",
"properties": {
"annotations": [],
"parameters": {},
"type": "AzureKeyVault",
"typeProperties": {
"baseUrl": "[reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName')), '2022-11-01').vaultUri]"
}
}
},
{
"type": "Microsoft.DataFactory/factories/linkedservices",
"apiVersion": "2018-06-01",
"name": "[format('{0}/{1}', parameters('dataFactoryName'), 'storage')]",
"properties": {
"annotations": [],
"parameters": {},
"type": "AzureBlobFS",
"typeProperties": {
"url": "[reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2022-09-01').primaryEndpoints.dfs]",
"accountKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "keyVault",
"type": "LinkedServiceReference"
},
"secretName": "[parameters('storageAccountName')]"
}
}
},
"dependsOn": [
"[resourceId('Microsoft.DataFactory/factories/linkedservices', parameters('dataFactoryName'), 'keyVault')]"
]
},
{
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"name": "[format('{0}/{1}', parameters('dataFactoryName'), variables('safeExportContainerName'))]",
"properties": {
"annotations": [],
"parameters": {
"fileName": {
"type": "String"
},
"folderName": {
"type": "String"
}
},
"type": "DelimitedText",
"typeProperties": "[union(variables('datasetPropsCommon'), variables('datasetPropsDelimitedText'), createObject('compressionCodec', 'none'))]",
"linkedServiceName": {
"parameters": {},
"referenceName": "storage",
"type": "LinkedServiceReference"
}
},
"dependsOn": [
"[resourceId('Microsoft.DataFactory/factories/linkedservices', parameters('dataFactoryName'), 'keyVault')]",
"[resourceId('Microsoft.DataFactory/factories/linkedservices', parameters('dataFactoryName'), 'storage')]"
]
},
{
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"name": "[format('{0}/{1}', parameters('dataFactoryName'), variables('safeIngestionContainerName'))]",
"properties": {
"annotations": [],
"parameters": {
"fileName": {
"type": "String"
},
"folderName": {
"type": "String"
}
},
"type": "[if(parameters('convertToParquet'), 'Parquet', 'DelimitedText')]",
"typeProperties": "[union(variables('datasetPropsCommon'), if(parameters('convertToParquet'), createObject(), variables('datasetPropsDelimitedText')), createObject('compressionCodec', 'gzip'))]",
"linkedServiceName": {
"parameters": {},
"referenceName": "storage",
"type": "LinkedServiceReference"
}
},
"dependsOn": [
"[resourceId('Microsoft.DataFactory/factories/linkedservices', parameters('dataFactoryName'), 'keyVault')]",
"[resourceId('Microsoft.DataFactory/factories/linkedservices', parameters('dataFactoryName'), 'storage')]"
]
},
{
"type": "Microsoft.DataFactory/factories/triggers",
"apiVersion": "2018-06-01",
"name": "[format('{0}/{1}', parameters('dataFactoryName'), variables('exportFileAddedTriggerName'))]",
"properties": {
"annotations": [],
"pipelines": [
{
"pipelineReference": {
"referenceName": "[format('{0}_ExecuteETL', parameters('exportContainerName'))]",
"type": "PipelineReference"
},
"parameters": {
"folderName": "@triggerBody().folderPath",
"fileName": "@triggerBody().fileName"
}
}
],
"type": "BlobEventsTrigger",
"typeProperties": {
"blobPathBeginsWith": "[format('/{0}/blobs/', parameters('exportContainerName'))]",
"blobPathEndsWith": ".csv",
"ignoreEmptyBlobs": true,
"scope": "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName'))]",
"events": [
"Microsoft.Storage.BlobCreated"
]
}
},
"dependsOn": [
"[resourceId('Microsoft.DataFactory/factories/pipelines', parameters('dataFactoryName'), format('{0}_ExecuteETL', variables('safeExportContainerName')))]",
"[resourceId('Microsoft.Resources/deploymentScripts', format('{0}_stopHubTriggers', parameters('dataFactoryName')))]"
]
},
{
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"name": "[format('{0}/{1}', parameters('dataFactoryName'), format('{0}_ExecuteETL', variables('safeExportContainerName')))]",
"properties": {
"activities": [
{
"name": "Execute",
"type": "ExecutePipeline",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "[format('{0}_ETL_{1}', variables('safeExportContainerName'), variables('safeIngestionContainerName'))]",
"type": "PipelineReference"
},
"waitOnCompletion": false,
"parameters": {
"folderName": {
"value": "@pipeline().parameters.folderName",
"type": "Expression"
},
"fileName": {
"value": "@pipeline().parameters.fileName",
"type": "Expression"
}
}
}
}
],
"parameters": {
"folderName": {
"type": "string"
},
"fileName": {
"type": "string"
}
},
"annotations": []
},
"dependsOn": [
"[resourceId('Microsoft.DataFactory/factories/pipelines', parameters('dataFactoryName'), format('{0}_ETL_{1}', variables('safeExportContainerName'), variables('safeIngestionContainerName')))]"
]
},
{
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"name": "[format('{0}/{1}', parameters('dataFactoryName'), format('{0}_ETL_{1}', variables('safeExportContainerName'), variables('safeIngestionContainerName')))]",
"properties": {
"activities": [
{
"name": "Wait",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 60
}
},
{
"name": "Set FolderArray",
"type": "SetVariable",
"dependsOn": [
{
"activity": "Wait",
"dependencyConditions": [
"Completed"
]
}
],
"userProperties": [],
"typeProperties": {
"variableName": "folderArray",
"value": {
"value": "@split(pipeline().parameters.folderName, '/')",
"type": "Expression"
}
}
},
{
"name": "Set FolderCount",
"type": "SetVariable",
"dependsOn": [
{
"activity": "Set FolderArray",
"dependencyConditions": [
"Completed"
]
}
],
"policy": {
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"variableName": "folderCount",
"value": "@length(split(pipeline().parameters.folderName, '/'))"
}
},
{
"name": "Set SecondToLastFolder",
"type": "SetVariable",
"dependsOn": [
{
"activity": "Set FolderCount",
"dependencyConditions": [
"Completed"
]
}
],
"policy": {
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"variableName": "secondToLastFolder",
"value": "@variables('folderArray')[sub(variables('folderCount'), 2)]"
}
},
{
"name": "Set ThirdToLastFolder",
"type": "SetVariable",
"dependsOn": [
{
"activity": "Set SecondToLastFolder",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"variableName": "thirdToLastFolder",
"value": "@variables('folderArray')[sub(variables('folderCount'), 3)]"
}
},
{
"name": "Set FourthToLastFolder",
"type": "SetVariable",
"dependsOn": [
{
"activity": "Set ThirdToLastFolder",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"variableName": "fourthToLastFolder",
"value": "@variables('folderArray')[sub(variables('folderCount'), 4)]"
}
},
{
"name": "Set Scope",
"type": "SetVariable",
"dependsOn": [
{
"activity": "Set FourthToLastFolder",
"dependencyConditions": [
"Completed"
]
}
],
"userProperties": [],
"typeProperties": {
"variableName": "scope",
"value": {
"value": "[format('@replace(split(pipeline().parameters.folderName, if(greater(length(variables(''secondToLastFolder'')), 12), variables(''thirdToLastFolder''), variables(''fourthToLastFolder'')))[0], ''{0}'', ''{1}'')', parameters('exportContainerName'), parameters('ingestionContainerName'))]",
"type": "Expression"
}
}
},
{
"name": "Set Metric",
"type": "SetVariable",
"dependsOn": [
{
"activity": "Set Scope",
"dependencyConditions": [
"Completed"
]
}
],
"userProperties": [],
"typeProperties": {
"variableName": "metric",
"value": {
"value": "focuscost",
"type": "Expression"
}
}
},
{
"name": "Set Date",
"type": "SetVariable",
"dependsOn": [
{
"activity": "Set Metric",
"dependencyConditions": [
"Completed"
]
}
],
"userProperties": [],
"typeProperties": {
"variableName": "date",
"value": {
"value": "@substring(if(greater(length(variables('secondToLastFolder')), 12), variables('secondToLastFolder'), variables('thirdToLastFolder')), 0, 6)",
"type": "Expression"
}
}
},
{
"name": "Set Destination File Name",
"description": "",
"type": "SetVariable",
"dependsOn": [
{
"activity": "Set Date",
"dependencyConditions": [
"Completed"
]
}
],
"userProperties": [],
"typeProperties": {
"variableName": "destinationFile",
"value": {
"value": "[format('@replace(pipeline().parameters.fileName, ''.csv'', ''{0}'')', if(parameters('convertToParquet'), '.parquet', '.csv.gz'))]",
"type": "Expression"
}
}
},
{
"name": "Set Destination Folder Name",
"type": "SetVariable",
"dependsOn": [
{
"activity": "Set Destination File Name",
"dependencyConditions": [
"Completed"
]
}
],
"userProperties": [],
"typeProperties": {
"variableName": "destinationFolder",
"value": {
"value": "@replace(concat(variables('scope'),variables('date'),'/',variables('metric')),'//','/')",
"type": "Expression"
}
}
},
{
"name": "Delete Target",
"type": "Delete",
"dependsOn": [
{
"activity": "Set Destination Folder Name",
"dependencyConditions": [
"Completed"
]
}
],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "[variables('safeIngestionContainerName')]",
"type": "DatasetReference",
"parameters": {
"folderName": {
"value": "@variables('destinationFolder')",
"type": "Expression"
},
"fileName": {
"value": "@variables('destinationFile')",
"type": "Expression"
}
}
},
"enableLogging": false,
"storeSettings": {
"type": "AzureBlobFSReadSettings",
"recursive": true,
"enablePartitionDiscovery": false
}
}
},
{
"name": "Convert CSV",
"type": "Copy",
"dependsOn": [
{
"activity": "Delete Target",
"dependencyConditions": [
"Completed"
]
}
],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "AzureBlobFSReadSettings",
"recursive": true,
"enablePartitionDiscovery": false
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": "[if(parameters('convertToParquet'), createObject('type', 'ParquetWriteSettings', 'fileExtension', '.parquet'), createObject('type', 'DelimitedTextWriteSettings', 'quoteAllText', true(), 'fileExtension', '.csv.gz'))]"
},
"enableStaging": false,
"parallelCopies": 1,
"validateDataConsistency": false,
"translator": {
"type": "TabularTranslator",
"mappings": "[variables('focusCostMappings')]"
}
},
"inputs": [
{
"referenceName": "[variables('safeExportContainerName')]",
"type": "DatasetReference",
"parameters": {
"folderName": {
"value": "@pipeline().parameters.folderName",
"type": "Expression"
},
"fileName": {
"value": "@pipeline().parameters.fileName",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "[variables('safeIngestionContainerName')]",
"type": "DatasetReference",
"parameters": {
"folderName": {
"value": "@variables('destinationFolder')",
"type": "Expression"
},
"fileName": {
"value": "@variables('destinationFile')",
"type": "Expression"
}
}
}
]
},
{
"name": "Delete CSV",
"type": "Delete",
"dependsOn": [
{
"activity": "Convert CSV",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "[variables('safeExportContainerName')]",
"type": "DatasetReference",
"parameters": {
"folderName": {
"value": "@pipeline().parameters.folderName",
"type": "Expression"
},
"fileName": {
"value": "@pipeline().parameters.fileName",
"type": "Expression"
}
}
},
"enableLogging": false,
"storeSettings": {
"type": "AzureBlobFSReadSettings",
"recursive": true,
"enablePartitionDiscovery": false
}
}
}
],
"parameters": {
"fileName": {
"type": "string"
},
"folderName": {
"type": "string"
}
},
"variables": {
"destinationFile": {
"type": "String"
},
"destinationFolder": {
"type": "String"
},
"folderArray": {
"type": "Array"
},
"folderCount": {
"type": "Integer"
},
"secondToLastFolder": {
"type": "String"
},
"thirdToLastFolder": {
"type": "String"
},
"fourthToLastFolder": {
"type": "String"
},
"scope": {
"type": "String"
},
"date": {
"type": "String"
},
"metric": {
"type": "String"
}
},
"annotations": []
},
"dependsOn": [
"[resourceId('Microsoft.DataFactory/factories/datasets', parameters('dataFactoryName'), variables('safeIngestionContainerName'))]",
"[resourceId('Microsoft.DataFactory/factories/datasets', parameters('dataFactoryName'), variables('safeExportContainerName'))]"
]
},
{
"type": "Microsoft.Resources/deploymentScripts",
"apiVersion": "2020-10-01",
"name": "[format('{0}_startHubTriggers', parameters('dataFactoryName'))]",
"location": "[if(startsWith(parameters('location'), 'china'), 'chinaeast2', parameters('location'))]",
"tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.Resources/deploymentScripts'), parameters('tagsByResource')['Microsoft.Resources/deploymentScripts'], createObject()))]",
"identity": {
"type": "UserAssigned",
"userAssignedIdentities": {
"[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName'))))]": {}
}
},
"kind": "AzurePowerShell",
"properties": {
"azPowerShellVersion": "8.0",
"retentionInterval": "PT1H",
"cleanupPreference": "OnSuccess",
"scriptContent": "[variables('$fxv#2')]",
"environmentVariables": [
{
"name": "DataFactorySubscriptionId",
"value": "[subscription().id]"
},
{
"name": "DataFactoryResourceGroup",
"value": "[resourceGroup().name]"
},
{
"name": "DataFactoryName",
"value": "[parameters('dataFactoryName')]"
},
{
"name": "Triggers",
"value": "[join(variables('allHubTriggers'), '|')]"
}
]
},
"dependsOn": [
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName')))]",
"identityRoleAssignments",
"[resourceId('Microsoft.DataFactory/factories/triggers', parameters('dataFactoryName'), variables('exportFileAddedTriggerName'))]"
]
}
],
"outputs": {
"resourceId": {
"type": "string",
"metadata": {
"description": "The Resource ID of the Data factory."
},
"value": "[resourceId('Microsoft.DataFactory/factories', parameters('dataFactoryName'))]"
},
"name": {
"type": "string",
"metadata": {
"description": "The Name of the Azure Data Factory instance."
},
"value": "[parameters('dataFactoryName')]"
}
}
}
},
"dependsOn": [
"[resourceId('Microsoft.Resources/deployments', 'keyVault')]",
"[resourceId('Microsoft.Resources/deployments', 'storage')]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2022-09-01",
"name": "keyVault",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"hubName": {
"value": "[parameters('hubName')]"
},
"uniqueSuffix": {
"value": "[variables('uniqueSuffix')]"
},
"location": {
"value": "[parameters('location')]"
},
"tags": {
"value": "[variables('resourceTags')]"
},
"tagsByResource": {
"value": "[parameters('tagsByResource')]"
},
"storageAccountName": {
"value": "[reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.name.value]"
},
"accessPolicies": {
"value": [
{
"objectId": "[reference(resourceId('Microsoft.DataFactory/factories', variables('dataFactoryName')), '2018-06-01', 'full').identity.principalId]",
"tenantId": "[subscription().tenantId]",
"permissions": {
"secrets": [
"get"
]
}
}
]
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.24.24.22086",
"templateHash": "10770478197596540923"
}
},
"parameters": {
"hubName": {
"type": "string",
"metadata": {
"description": "Required. Name of the hub. Used to ensure unique resource names."
}
},
"uniqueSuffix": {
"type": "string",
"metadata": {
"description": "Required. Suffix to add to the KeyVault instance name to ensure uniqueness."
}
},
"location": {
"type": "string",
"defaultValue": "[resourceGroup().location]",
"metadata": {
"description": "Optional. Location for all resources."
}
},
"accessPolicies": {
"type": "array",
"defaultValue": [],
"metadata": {
"description": "Optional. Array of access policies object."
}
},
"storageAccountName": {
"type": "string",
"metadata": {
"description": "Required. Name of the storage account to store access keys for."
}
},
"sku": {
"type": "string",
"defaultValue": "premium",
"allowedValues": [
"premium",
"standard"
],
"metadata": {
"description": "Optional. Specifies the SKU for the vault."
}
},
"tags": {
"type": "object",
"defaultValue": {},
"metadata": {
"description": "Optional. Resource tags."
}
},
"tagsByResource": {
"type": "object",
"defaultValue": {},
"metadata": {
"description": "Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources."
}
}
},
"variables": {
"copy": [
{
"name": "formattedAccessPolicies",
"count": "[length(parameters('accessPolicies'))]",
"input": {
"applicationId": "[if(contains(parameters('accessPolicies')[copyIndex('formattedAccessPolicies')], 'applicationId'), parameters('accessPolicies')[copyIndex('formattedAccessPolicies')].applicationId, '')]",
"objectId": "[if(contains(parameters('accessPolicies')[copyIndex('formattedAccessPolicies')], 'objectId'), parameters('accessPolicies')[copyIndex('formattedAccessPolicies')].objectId, '')]",
"permissions": "[parameters('accessPolicies')[copyIndex('formattedAccessPolicies')].permissions]",
"tenantId": "[if(contains(parameters('accessPolicies')[copyIndex('formattedAccessPolicies')], 'tenantId'), parameters('accessPolicies')[copyIndex('formattedAccessPolicies')].tenantId, tenant().tenantId)]"
}
}
],
"keyVaultPrefix": "[format('{0}-vault', replace(parameters('hubName'), '_', '-'))]",
"keyVaultSuffix": "[format('-{0}', parameters('uniqueSuffix'))]",
"keyVaultName": "[replace(format('{0}{1}', take(variables('keyVaultPrefix'), sub(24, length(variables('keyVaultSuffix')))), variables('keyVaultSuffix')), '--', '-')]"
},
"resources": [
{
"type": "Microsoft.KeyVault/vaults",
"apiVersion": "2022-11-01",
"name": "[variables('keyVaultName')]",
"location": "[parameters('location')]",
"tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.KeyVault/vaults'), parameters('tagsByResource')['Microsoft.KeyVault/vaults'], createObject()))]",
"properties": {
"enabledForDeployment": true,
"enabledForTemplateDeployment": true,
"enabledForDiskEncryption": true,
"enableSoftDelete": true,
"softDeleteRetentionInDays": 90,
"enableRbacAuthorization": false,
"createMode": "default",
"tenantId": "[subscription().tenantId]",
"accessPolicies": "[variables('formattedAccessPolicies')]",
"sku": {
"name": "[if(startsWith(parameters('location'), 'china'), 'standard', parameters('sku'))]",
"family": "A"
}
}
},
{
"condition": "[not(empty(parameters('accessPolicies')))]",
"type": "Microsoft.KeyVault/vaults/accessPolicies",
"apiVersion": "2022-11-01",
"name": "[format('{0}/{1}', variables('keyVaultName'), 'add')]",
"properties": {
"accessPolicies": "[variables('formattedAccessPolicies')]"
},
"dependsOn": [
"[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]"
]
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2022-11-01",
"name": "[format('{0}/{1}', variables('keyVaultName'), parameters('storageAccountName'))]",
"properties": {
"attributes": {
"enabled": true,
"exp": 1702648632,
"nbf": 10000
},
"value": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2022-09-01').keys[0].value]"
},
"dependsOn": [
"[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]"
]
}
],
"outputs": {
"resourceId": {
"type": "string",
"metadata": {
"description": "The resource ID of the key vault."
},
"value": "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]"
},
"name": {
"type": "string",
"metadata": {
"description": "The name of the key vault."
},
"value": "[variables('keyVaultName')]"
},
"uri": {
"type": "string",
"metadata": {
"description": "The URI of the key vault."
},
"value": "[reference(resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName')), '2022-11-01').vaultUri]"
}
}
}
},
"dependsOn": [
"[resourceId('Microsoft.DataFactory/factories', variables('dataFactoryName'))]",
"[resourceId('Microsoft.Resources/deployments', 'storage')]"
]
}
],
"outputs": {
"name": {
"type": "string",
"metadata": {
"description": "Name of the deployed hub instance."
},
"value": "[parameters('hubName')]"
},
"location": {
"type": "string",
"metadata": {
"description": "Azure resource location resources were deployed to."
},
"value": "[parameters('location')]"
},
"dataFactorytName": {
"type": "string",
"metadata": {
"description": "Name of the Data Factory."
},
"value": "[variables('dataFactoryName')]"
},
"storageAccountId": {
"type": "string",
"metadata": {
"description": "Resource ID of the storage account created for the hub instance. This must be used when creating the Cost Management export."
},
"value": "[reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.resourceId.value]"
},
"storageAccountName": {
"type": "string",
"metadata": {
"description": "Name of the storage account created for the hub instance. This must be used when connecting FinOps toolkit Power BI reports to your data."
},
"value": "[reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.name.value]"
},
"storageUrlForPowerBI": {
"type": "string",
"metadata": {
"description": "URL to use when connecting custom Power BI reports to your data."
},
"value": "[format('https://{0}.dfs.{1}/{2}', reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.name.value, environment().suffixes.storage, reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.ingestionContainer.value)]"
}
}
}
}
}
],
"outputs": {
"name": {
"type": "string",
"metadata": {
"description": "The name of the resource group."
},
"value": "[parameters('hubName')]"
},
"location": {
"type": "string",
"metadata": {
"description": "The location the resources wer deployed to."
},
"value": "[parameters('location')]"
},
"dataFactorytName": {
"type": "string",
"metadata": {
"description": "Name of the Data Factory."
},
"value": "[reference(resourceId('Microsoft.Resources/deployments', 'hub'), '2022-09-01').outputs.dataFactorytName.value]"
},
"storageAccountId": {
"type": "string",
"metadata": {
"description": "The resource ID of the deployed storage account."
},
"value": "[reference(resourceId('Microsoft.Resources/deployments', 'hub'), '2022-09-01').outputs.storageAccountId.value]"
},
"storageAccountName": {
"type": "string",
"metadata": {
"description": "Name of the storage account created for the hub instance. This must be used when connecting FinOps toolkit Power BI reports to your data."
},
"value": "[reference(resourceId('Microsoft.Resources/deployments', 'hub'), '2022-09-01').outputs.storageAccountName.value]"
},
"storageUrlForPowerBI": {
"type": "string",
"metadata": {
"description": "URL to use when connecting custom Power BI reports to your data."
},
"value": "[reference(resourceId('Microsoft.Resources/deployments', 'hub'), '2022-09-01').outputs.storageUrlForPowerBI.value]"
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment