Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save zzzprojects/d730288aedee8e1895b1 to your computer and use it in GitHub Desktop.
Save zzzprojects/d730288aedee8e1895b1 to your computer and use it in GitHub Desktop.
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.SqlClient;
using System.Linq;
namespace Z.EntityFramework.Extensions
{
public class HowTo_BulkInsert_InSqlServer
{
public const int BatchSize = 4000;
public const string ConnectionString = @"Server=XPS8700;Initial Catalog=Z.Test.EntityFramework.Plus.EF6;Integrated Security=True;";
internal const string InsertStagingTable = @"MERGE INTO EntitySimples AS DestinationTable
USING
(
{StagingTable}
) AS StagingTable
ON 1 = 2
WHEN NOT MATCHED THEN
INSERT ( ColumnInt )
VALUES ( ColumnInt )
;";
internal const string InsertStagingTableWithOutput = @"MERGE INTO EntitySimples AS DestinationTable
USING
(
SELECT ZZZ_Index, ColumnInt FROM {TemporaryTable} WHERE ZZZ_Index >= @IndexStart AND ZZZ_Index < @IndexEnd
) AS StagingTable
ON 1 = 2
WHEN NOT MATCHED THEN
INSERT ( ColumnInt )
VALUES ( ColumnInt )
OUTPUT
StagingTable.ZZZ_Index,
INSERTED.ID
;";
public static void Test()
{
var list1 = GenerateItems(9);
var list2 = GenerateItems(10);
var list3 = GenerateItems(10);
BulkInsert(list1);
BulkInsert(list2);
BulkInsert(list3, true);
}
public static List<EntitySimple> GenerateItems(int count)
{
var list = new List<EntitySimple>();
for (var i = 0; i < count; i++)
{
list.Add(new EntitySimple {ColumnInt = i});
}
return list;
}
public static void BulkInsert(List<EntitySimple> list, bool outputIdentity = false)
{
if (outputIdentity)
{
BulkInsertTemporaryTable(list);
}
else if (list.Count >= 10)
{
BulkInsertBulkCopy(list);
}
else
{
BulkInsertSqlCommand(list);
}
}
private static void BulkInsertSqlCommand(List<EntitySimple> list)
{
// CREATE sql derived table
var sqlDerivedTable = "SELECT " + string.Join(" UNION SELECT ", list.Select((simple, i) => "@" + i + " AS ColumnInt"));
using (var connection = new SqlConnection(ConnectionString))
{
connection.Open();
using (var command = new SqlCommand(InsertStagingTable.Replace("{StagingTable}", sqlDerivedTable), connection))
{
// ADD parameter
for (var i = 0; i < list.Count; i++)
{
command.Parameters.AddWithValue("@" + i, list[i].ColumnInt);
}
command.ExecuteNonQuery();
}
}
}
private static void BulkInsertBulkCopy(List<EntitySimple> list)
{
var dt = ToDataTable(list);
using (var connection = new SqlConnection(ConnectionString))
{
connection.Open();
using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.CheckConstraints | SqlBulkCopyOptions.FireTriggers, null))
{
bulkCopy.BatchSize = BatchSize;
bulkCopy.ColumnMappings.Add("ColumnInt", "ColumnInt");
bulkCopy.DestinationTableName = "EntitySimples";
bulkCopy.WriteToServer(dt);
}
}
}
private static void BulkInsertTemporaryTable(List<EntitySimple> list)
{
var dt = ToDataTable(list);
var temporaryTable = "#" + Guid.NewGuid().ToString().Replace("-", "_");
using (var connection = new SqlConnection(ConnectionString))
{
connection.Open();
// CREATE temporary table
using (var command = new SqlCommand("CREATE TABLE " + temporaryTable + " (ZZZ_Index INT, ColumnInt INT)", connection))
{
command.ExecuteNonQuery();
}
// INSERT INTO temporary table
using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.CheckConstraints | SqlBulkCopyOptions.FireTriggers, null))
{
bulkCopy.BatchSize = BatchSize;
bulkCopy.ColumnMappings.Add("ZZZ_Index", "ZZZ_Index");
bulkCopy.ColumnMappings.Add("ColumnInt", "ColumnInt");
bulkCopy.DestinationTableName = temporaryTable;
bulkCopy.WriteToServer(dt);
}
// INSERT INTO destination FROM temporary table
var currentIndex = 0;
while (currentIndex < list.Count)
{
using (var command = new SqlCommand(InsertStagingTableWithOutput.Replace("{TemporaryTable}", temporaryTable), connection))
{
command.Parameters.AddWithValue(@"IndexStart", currentIndex);
command.Parameters.AddWithValue("@IndexEnd", currentIndex + BatchSize);
var dtOutput = new DataTable();
using (var adapter = new SqlDataAdapter(command))
{
adapter.Fill(dtOutput);
}
// OUTPUT value
foreach (DataRow dr in dtOutput.Rows)
{
var index = Convert.ToInt32(dr["ZZZ_Index"]);
list[index].ID = Convert.ToInt32(dr["ID"]);
}
}
currentIndex += BatchSize;
}
}
}
public static DataTable ToDataTable(List<EntitySimple> list)
{
// CREATE table
var dt = new DataTable();
// ADD columns
dt.Columns.Add("ZZZ_Index", typeof (int));
dt.Columns.Add("ColumnInt", typeof (int));
// ADD rows
for (var i = 0; i < list.Count; i++)
{
dt.Rows.Add(i, list[i].ColumnInt);
}
return dt;
}
public class EntitySimple
{
public int ID { get; set; }
public int ColumnInt { get; set; }
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment