Created
May 21, 2023 11:04
-
-
Save Ricky-G/a9d670728a4f554b1234e4cb3ee74189 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System; | |
using System.IO; | |
using System.Threading.Tasks; | |
using Microsoft.Azure.Storage; | |
using Microsoft.Azure.Storage.Blob; | |
using Microsoft.Azure.WebJobs; | |
using Microsoft.Extensions.Logging; | |
using System.Diagnostics; | |
using System.IO.Compression; | |
namespace FileUnzipFunction | |
{ | |
public class FileUnzipBlobTrigger | |
{ | |
private static readonly CloudBlobClient _blobClient; | |
static FileUnzipBlobTrigger() | |
{ | |
string storageConnectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage"); | |
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString); | |
_blobClient = storageAccount.CreateCloudBlobClient(); | |
} | |
[FunctionName("FileUnzipBlobTrigger")] | |
public static async Task Run([BlobTrigger("zipped/{name}", Connection = "AzureWebJobsStorage")] Stream inputBlob, string name, ILogger log) | |
{ | |
log.LogInformation($"Processing blob: {name}"); | |
LogCurrentMemoryConsumption(log); | |
log.LogInformation($"Processing blob: {name}"); | |
var unzipContainer = _blobClient.GetContainerReference("unzipped"); | |
using (ZipArchive archive = new ZipArchive(inputBlob)) | |
{ | |
foreach (ZipArchiveEntry entry in archive.Entries) | |
{ | |
var blob = unzipContainer.GetBlockBlobReference(entry.FullName); | |
using (var stream = entry.Open()) | |
{ | |
log.LogInformation($"Unzipping {entry.FullName}"); | |
LogCurrentMemoryConsumption(log); | |
await blob.UploadFromStreamAsync(stream); | |
} | |
} | |
} | |
} | |
public static void LogCurrentMemoryConsumption(ILogger log) | |
{ | |
var process = Process.GetCurrentProcess(); | |
var physicalMemoryUsage = process.WorkingSet64 / 1024.0 / 1024.0 / 1024.0; | |
var virtualMemorySize = process.VirtualMemorySize64 / 1024.0 / 1024.0 / 1024.0; | |
var pagedMemorySize = process.PagedMemorySize64 / 1024.0 / 1024.0 / 1024.0; | |
log.LogInformation($"Memory Usage: WorkingSet64={physicalMemoryUsage} MB, VirtualMemorySize64={virtualMemorySize} MB, PagedMemorySize64={pagedMemorySize} MB"); | |
} | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System; | |
using System.IO; | |
using System.Threading.Tasks; | |
using Microsoft.Azure.Storage; | |
using Microsoft.Azure.Storage.Blob; | |
using Microsoft.Azure.WebJobs; | |
using Microsoft.Extensions.Logging; | |
using ICSharpCode.SharpZipLib.Zip; | |
using ICSharpCode.SharpZipLib.Core; | |
using System.Diagnostics; | |
namespace FileUnzipFunction | |
{ | |
public class FileUnzipBlobTrigger | |
{ | |
private static readonly CloudBlobClient _blobClient; | |
static FileUnzipBlobTrigger() | |
{ | |
string storageConnectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage"); | |
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString); | |
_blobClient = storageAccount.CreateCloudBlobClient(); | |
} | |
[FunctionName("FileUnzipBlobTrigger")] | |
public static async Task Run([BlobTrigger("zipped/{name}", Connection = "AzureWebJobsStorage")] Stream inputBlob, string name, ILogger log) | |
{ | |
log.LogInformation($"Processing blob: {name}"); | |
LogCurrentMemoryConsumption(log); | |
var unzipContainer = _blobClient.GetContainerReference("unzipped"); | |
using (var zipInputStream = new ZipInputStream(inputBlob)) | |
{ | |
ZipEntry entry; | |
while ((entry = zipInputStream.GetNextEntry()) != null) | |
{ | |
if (!entry.IsFile) continue; // Ignore directories | |
var blob = unzipContainer.GetBlockBlobReference(entry.Name); | |
log.LogInformation($"Unzipping {entry.Name}"); | |
LogCurrentMemoryConsumption(log); | |
// Copy the entry's data from the zip input stream to the blob output stream | |
using (var blobStream = await blob.OpenWriteAsync()) | |
{ | |
StreamUtils.Copy(zipInputStream, blobStream, new byte[4096]); | |
} | |
LogCurrentMemoryConsumption(log); | |
} | |
} | |
} | |
public static void LogCurrentMemoryConsumption(ILogger log) | |
{ | |
var process = Process.GetCurrentProcess(); | |
var physicalMemoryUsage = process.WorkingSet64 / 1024.0 / 1024.0 / 1024.0; | |
var virtualMemorySize = process.VirtualMemorySize64 / 1024.0 / 1024.0 / 1024.0; | |
var pagedMemorySize = process.PagedMemorySize64 / 1024.0 / 1024.0 / 1024.0; | |
log.LogInformation($"Memory Usage: WorkingSet64={physicalMemoryUsage} MB, VirtualMemorySize64={virtualMemorySize} MB, PagedMemorySize64={pagedMemorySize} MB"); | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
The two code samples are actually quite similar in their basic approach. They both handle the data in a streaming manner, which allows them to deal with large files without consuming a lot of memory.
However, there are some differences in the details of how they handle the streaming, which may have implications for their performance and resource usage: