Last active
December 28, 2015 11:49
-
-
Save troyhunt/7496278 to your computer and use it in GitHub Desktop.
Does this code demonstrate the absolute max possible writes per second to an Azure storage table? I'm seeing it return averages of about 900 to 1,000 rows per second by batching inserts into 100 rows a go. Short of running batches async, is this code already optimal? Now revised to run each partition's batches in async. Throughput has jumped up …
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System; | |
using System.Collections.Generic; | |
using System.Configuration; | |
using System.Diagnostics; | |
using System.Net; | |
using System.Threading.Tasks; | |
using Microsoft.WindowsAzure.Storage; | |
using Microsoft.WindowsAzure.Storage.Table; | |
namespace AzureSpeedTest | |
{ | |
class Program | |
{ | |
private const int Partitions = 10; | |
private const int RowsPerPartition = 1000; | |
private const int RowsPerBatch = 100; | |
static void Main() | |
{ | |
var connString = ConfigurationManager.ConnectionStrings["StorageConnectionString"].ConnectionString; | |
var storageAccount = CloudStorageAccount.Parse(connString); | |
ServicePointManager.UseNagleAlgorithm = false; | |
var tableClient = storageAccount.CreateCloudTableClient(); | |
var table = tableClient.GetTableReference("AzureSpeedTest"); | |
table.CreateIfNotExists(); | |
Console.WriteLine("Starting..."); | |
var sw = new Stopwatch(); | |
sw.Start(); | |
for (var partition = 0; partition < Partitions; partition++) | |
{ | |
Console.WriteLine("Partition {0}", partition); | |
BatchInsert(table, partition).Wait(); | |
} | |
sw.Stop(); | |
const int totalRows = Partitions * RowsPerPartition; | |
var elapsedSeconds = sw.ElapsedMilliseconds / 1000; | |
Console.WriteLine("Inserted {0:n0} rows across {1:n0} partitions in {2:n0} seconds at {3:n0} rows per second.", | |
totalRows, Partitions, elapsedSeconds, totalRows / elapsedSeconds); | |
Console.ReadLine(); | |
} | |
private static async Task BatchInsert(CloudTable table, int partition) | |
{ | |
var tasks = new List<Task>(); | |
for (var row = 0; row <= RowsPerPartition / RowsPerBatch; row++) | |
{ | |
var currentRow = row; | |
var task = Task.Factory.StartNew(() => | |
{ | |
var batchOperation = new TableBatchOperation(); | |
for (var batchRow = currentRow * RowsPerBatch; batchRow < (currentRow * RowsPerBatch) + RowsPerBatch; batchRow++) | |
{ | |
batchOperation.Insert(new TestEntity { PartitionKey = partition.ToString(), RowKey = batchRow.ToString() }); | |
} | |
table.ExecuteBatch(batchOperation); | |
}); | |
tasks.Add(task); | |
} | |
await Task.WhenAll(tasks); | |
} | |
public class TestEntity : TableEntity | |
{ | |
public TestEntity() { } | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Don't forget to turn off "Expect100Continue" and increase the connection limit (it's 2 by default).