Skip to content

Instantly share code, notes, and snippets.

@itn3000
Last active April 14, 2016 07:19
Show Gist options
  • Save itn3000/74414b0f8f528a1f48a27cf23f8771d1 to your computer and use it in GitHub Desktop.
Save itn3000/74414b0f8f528a1f48a27cf23f8771d1 to your computer and use it in GitHub Desktop.
Npgsql bulk insert with binary format and typemap
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Npgsql;
using System.Linq.Expressions;
using System.Reflection;
namespace LambdaTest
{
using CacheFuncType = System.Action<NpgsqlBinaryImporter, object>;
static class Class1
{
static Dictionary<Type, CacheFuncType> FuncCache = new Dictionary<Type, CacheFuncType>();
static Dictionary<Type, object> CreateValuemap(long idx)
{
return new Dictionary<Type, object>()
{
{ typeof(long),idx }
,
{ typeof(string),"mogemoge"+idx.ToString() }
,
{ typeof(DateTimeOffset),DateTimeOffset.Now }
};
}
static Expression<CacheFuncType> CreateWriteLambda(Type t)
{
var source = Expression.Parameter(typeof(object), "source");
var objparam = Expression.Parameter(typeof(NpgsqlBinaryImporter), "writer");
var emethod = Expression.Call(objparam, "Write", new Type[] { t }, Expression.Convert(source, t));
return Expression.Lambda<CacheFuncType>(emethod, objparam, source);
}
static void BulkInsertTest(NpgsqlConnection con)
{
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
using (var writer = con.BeginBinaryImport("copy test2(a,b,c) from stdin(format binary)"))
{
for (int i = 0; i < 1000000; i++)
{
writer.StartRow();
//writer.Write((long)i);
//writer.Write(string.Format("mogemoge{0}", i));
//writer.Write(DateTimeOffset.Now);
var valueMap = CreateValuemap(i);
foreach (var kv in valueMap)
{
if (!FuncCache.ContainsKey(kv.Key))
{
var lambda = CreateWriteLambda(kv.Key);
FuncCache[kv.Key] = lambda.Compile();
}
FuncCache[kv.Key].Invoke(writer, kv.Value);
}
}
}
Console.WriteLine($"{sw.Elapsed}");
}
public static void Test()
{
var cb = new NpgsqlConnectionStringBuilder();
cb.Host = "localhost";
cb.Port = 5432;
cb.Database = "test";
cb.Username = "testuser";
cb.Password = "hogehoge";
cb.CommandTimeout = 0;
using (var con = new NpgsqlConnection(cb))
{
con.Open();
using (var cmd = con.CreateCommand())
{
cmd.CommandText = "truncate table test2";
cmd.ExecuteNonQuery();
}
BulkInsertTest(con);
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment