This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using Azure.Storage.Blobs; | |
using System.IO; | |
static internal MemoryStream GetBlobStream(string connectionString, string blobContainer, string blobName) | |
{ | |
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString); | |
BlobClient blobClient = new BlobClient(connectionString, blobContainer, blobName); | |
var memoryStream = new MemoryStream(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using ChoETL; | |
if (fileType.ToUpper().Equals("PARQUET")) | |
{ | |
memoryStream.Seek(0, SeekOrigin.Begin); | |
int recordsRead = 0; | |
// loop through the rows of the file | |
foreach (dynamic e in new ChoParquetReader(memoryStream)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System.Dynamic; | |
dynamic record = new ExpandoObject(); | |
AddProperty(record, "property_name", "property_value"); | |
public static void AddProperty(ExpandoObject expando, string propertyName, object propertyValue) | |
{ | |
var expandoDict = expando as IDictionary<string, object>; | |
if (expandoDict.ContainsKey(propertyName)) | |
expandoDict[propertyName] = propertyValue; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using Azure.Storage.Blobs; | |
using System.IO; | |
static internal MemoryStream GetBlobStreamUsingSAS(string AccountName , string SAS, string Container, string blobName) | |
{ | |
Uri uriAddress = new Uri("https://"+AccountName+ ".blob.core.windows.net/" + SAS); | |
var newurl = uriAddress.Scheme + "://" + uriAddress.Host + "/" + Container + "/" + blobName + uriAddress.Query; | |
BlobClient blobClient = new BlobClient(new Uri(newurl), null); | |
var memoryStream = new MemoryStream(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using Microsoft.Azure.Cosmos; | |
// DB Level Throughput (Shared among collections) , returns both Manual and Auto Scale Throughput | |
public static async Task<int?[]> GetDBThroughput(string connection, string key, string db) | |
{ | |
using (var clientcos = new CosmosClient(connection, key)) | |
{ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from azure.storage.blob.aio import BlobClient | |
account_name = "{account_name}" | |
target_conn_str = "{connection_string}" | |
target_container_name = "{container_name}" | |
target_blob_name = "{file_to_upload}" | |
blob = BlobClient.from_connection_string(conn_str=target_conn_str, container_name=target_container_name, blob_name= target_blob_name) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# set the schema with the following json structure | |
jsonStringFromFile=""" | |
{ | |
"type": "struct", | |
"fields": [ | |
{ | |
"name": "id", | |
"type": "integer", | |
"nullable": true, | |
"metadata": {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# set the varilable in python | |
filePath='abfss://{container_name}@{storage_account_name}.dfs.core.windows.net/curated/Dims/Dim_X/*.csv' | |
spark.conf.set('f.filePath',filePath) | |
# use the varilable in SQL command | |
%sql | |
CREATE OR REPLACE TEMPORARY VIEW V_SomeView | |
USING CSV |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# create DF from Table/View: | |
df=spark.sql("select * from {table_view_name}") | |
#create View from DF: | |
df.createOrReplaceTempView("{view_name}") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# set access info , disclaimer: this is not the best way to access your data from security perspective. | |
spark.conf.set( | |
"fs.azure.account.key.{storage_account_name}.dfs.core.windows.net", | |
"{storage_key_here}" | |
) | |
import datetime | |
now = datetime.datetime.now() |
OlderNewer