Skip to content

Instantly share code, notes, and snippets.

@MaximRouiller
Last active January 17, 2023 02:58
Show Gist options
  • Star 2 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save MaximRouiller/b5613327a28ae9e052c0425f636ffc34 to your computer and use it in GitHub Desktop.
Save MaximRouiller/b5613327a28ae9e052c0425f636ffc34 to your computer and use it in GitHub Desktop.
Automatically download files to Azure Blob Storage using Copy From URL API with C#
using Azure.Storage.Blobs;
using System;
using System.IO;
using System.Threading.Tasks;
namespace ConsoleApp2
{
class Program
{
// Required .NET Package: Azure.Storage.Blobs version >=12
// API Docs: https://docs.microsoft.com/en-us/rest/api/storageservices/copy-blob-from-url
static async Task Main(string[] args)
{
string connectionString = Environment.GetEnvironmentVariable("AZURE_CONNECTION_STRING", EnvironmentVariableTarget.Process);
var filesToProcess = new[] {
"http://deepyeti.ucsd.edu/jianmo/amazon/categoryFilesSmall/Grocery_and_Gourmet_Food.csv", // ~200MB
"http://dmery.sitios.ing.uc.cl/images/GDXray/Castings.zip", // ~295MB
"http://dmery.sitios.ing.uc.cl/images/GDXray/Baggages.zip" // ~3GB
};
Console.WriteLine("Copying Blob from URL (size limited to 250MB (?) )");
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
string containerName = $"data-{DateTime.UtcNow:yyyyMMddHHmm}";
BlobContainerClient containerClient = await blobServiceClient.CreateBlobContainerAsync(containerName);
await containerClient.CreateIfNotExistsAsync();
foreach (var fileToProcess in filesToProcess)
{
string fileName = Path.GetFileName(fileToProcess);
var blobClient = containerClient.GetBlobClient(fileName);
blobClient.StartCopyFromUri(new Uri(fileToProcess));
}
Console.WriteLine($"Operation started. Closing application.");
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment