Last active
March 18, 2016 23:45
-
-
Save zzzprojects/d730288aedee8e1895b1 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System; | |
using System.Collections.Generic; | |
using System.Data; | |
using System.Data.SqlClient; | |
using System.Linq; | |
namespace Z.EntityFramework.Extensions | |
{ | |
public class HowTo_BulkInsert_InSqlServer | |
{ | |
public const int BatchSize = 4000; | |
public const string ConnectionString = @"Server=XPS8700;Initial Catalog=Z.Test.EntityFramework.Plus.EF6;Integrated Security=True;"; | |
internal const string InsertStagingTable = @"MERGE INTO EntitySimples AS DestinationTable | |
USING | |
( | |
{StagingTable} | |
) AS StagingTable | |
ON 1 = 2 | |
WHEN NOT MATCHED THEN | |
INSERT ( ColumnInt ) | |
VALUES ( ColumnInt ) | |
;"; | |
internal const string InsertStagingTableWithOutput = @"MERGE INTO EntitySimples AS DestinationTable | |
USING | |
( | |
SELECT ZZZ_Index, ColumnInt FROM {TemporaryTable} WHERE ZZZ_Index >= @IndexStart AND ZZZ_Index < @IndexEnd | |
) AS StagingTable | |
ON 1 = 2 | |
WHEN NOT MATCHED THEN | |
INSERT ( ColumnInt ) | |
VALUES ( ColumnInt ) | |
OUTPUT | |
StagingTable.ZZZ_Index, | |
INSERTED.ID | |
;"; | |
public static void Test() | |
{ | |
var list1 = GenerateItems(9); | |
var list2 = GenerateItems(10); | |
var list3 = GenerateItems(10); | |
BulkInsert(list1); | |
BulkInsert(list2); | |
BulkInsert(list3, true); | |
} | |
public static List<EntitySimple> GenerateItems(int count) | |
{ | |
var list = new List<EntitySimple>(); | |
for (var i = 0; i < count; i++) | |
{ | |
list.Add(new EntitySimple {ColumnInt = i}); | |
} | |
return list; | |
} | |
public static void BulkInsert(List<EntitySimple> list, bool outputIdentity = false) | |
{ | |
if (outputIdentity) | |
{ | |
BulkInsertTemporaryTable(list); | |
} | |
else if (list.Count >= 10) | |
{ | |
BulkInsertBulkCopy(list); | |
} | |
else | |
{ | |
BulkInsertSqlCommand(list); | |
} | |
} | |
private static void BulkInsertSqlCommand(List<EntitySimple> list) | |
{ | |
// CREATE sql derived table | |
var sqlDerivedTable = "SELECT " + string.Join(" UNION SELECT ", list.Select((simple, i) => "@" + i + " AS ColumnInt")); | |
using (var connection = new SqlConnection(ConnectionString)) | |
{ | |
connection.Open(); | |
using (var command = new SqlCommand(InsertStagingTable.Replace("{StagingTable}", sqlDerivedTable), connection)) | |
{ | |
// ADD parameter | |
for (var i = 0; i < list.Count; i++) | |
{ | |
command.Parameters.AddWithValue("@" + i, list[i].ColumnInt); | |
} | |
command.ExecuteNonQuery(); | |
} | |
} | |
} | |
private static void BulkInsertBulkCopy(List<EntitySimple> list) | |
{ | |
var dt = ToDataTable(list); | |
using (var connection = new SqlConnection(ConnectionString)) | |
{ | |
connection.Open(); | |
using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.CheckConstraints | SqlBulkCopyOptions.FireTriggers, null)) | |
{ | |
bulkCopy.BatchSize = BatchSize; | |
bulkCopy.ColumnMappings.Add("ColumnInt", "ColumnInt"); | |
bulkCopy.DestinationTableName = "EntitySimples"; | |
bulkCopy.WriteToServer(dt); | |
} | |
} | |
} | |
private static void BulkInsertTemporaryTable(List<EntitySimple> list) | |
{ | |
var dt = ToDataTable(list); | |
var temporaryTable = "#" + Guid.NewGuid().ToString().Replace("-", "_"); | |
using (var connection = new SqlConnection(ConnectionString)) | |
{ | |
connection.Open(); | |
// CREATE temporary table | |
using (var command = new SqlCommand("CREATE TABLE " + temporaryTable + " (ZZZ_Index INT, ColumnInt INT)", connection)) | |
{ | |
command.ExecuteNonQuery(); | |
} | |
// INSERT INTO temporary table | |
using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.CheckConstraints | SqlBulkCopyOptions.FireTriggers, null)) | |
{ | |
bulkCopy.BatchSize = BatchSize; | |
bulkCopy.ColumnMappings.Add("ZZZ_Index", "ZZZ_Index"); | |
bulkCopy.ColumnMappings.Add("ColumnInt", "ColumnInt"); | |
bulkCopy.DestinationTableName = temporaryTable; | |
bulkCopy.WriteToServer(dt); | |
} | |
// INSERT INTO destination FROM temporary table | |
var currentIndex = 0; | |
while (currentIndex < list.Count) | |
{ | |
using (var command = new SqlCommand(InsertStagingTableWithOutput.Replace("{TemporaryTable}", temporaryTable), connection)) | |
{ | |
command.Parameters.AddWithValue(@"IndexStart", currentIndex); | |
command.Parameters.AddWithValue("@IndexEnd", currentIndex + BatchSize); | |
var dtOutput = new DataTable(); | |
using (var adapter = new SqlDataAdapter(command)) | |
{ | |
adapter.Fill(dtOutput); | |
} | |
// OUTPUT value | |
foreach (DataRow dr in dtOutput.Rows) | |
{ | |
var index = Convert.ToInt32(dr["ZZZ_Index"]); | |
list[index].ID = Convert.ToInt32(dr["ID"]); | |
} | |
} | |
currentIndex += BatchSize; | |
} | |
} | |
} | |
public static DataTable ToDataTable(List<EntitySimple> list) | |
{ | |
// CREATE table | |
var dt = new DataTable(); | |
// ADD columns | |
dt.Columns.Add("ZZZ_Index", typeof (int)); | |
dt.Columns.Add("ColumnInt", typeof (int)); | |
// ADD rows | |
for (var i = 0; i < list.Count; i++) | |
{ | |
dt.Rows.Add(i, list[i].ColumnInt); | |
} | |
return dt; | |
} | |
public class EntitySimple | |
{ | |
public int ID { get; set; } | |
public int ColumnInt { get; set; } | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment