Skip to content

Instantly share code, notes, and snippets.

@MarkPflug
Created March 24, 2023 15:26
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save MarkPflug/7470c364eb94560b2ac347d2ee94274e to your computer and use it in GitHub Desktop.
Save MarkPflug/7470c364eb94560b2ac347d2ee94274e to your computer and use it in GitHub Desktop.
A low-allocation CSV struct writer
using System.Collections;
using System.Collections.ObjectModel;
using System.Data.Common;
using System.Diagnostics;
var rand = new Random();
var data =
Enumerable
.Range(0, 10000000)
.Select(
i =>
new MyData
{
Id = Guid.NewGuid(),
TimeStamp = DateTime.Now.AddDays(rand.NextDouble() * -10.0),
LocX = rand.NextDouble() * 1000.0,
LocY = rand.NextDouble() * 1000.0,
}
).ToArray();
var start = GC.GetTotalAllocatedBytes(true);
var sw = Stopwatch.StartNew();
using var csvWriter = Sylvan.Data.Csv.CsvDataWriter.Create("data.csv");
csvWriter.Write(new MyDataReader(data));
sw.Stop();
var end = GC.GetTotalAllocatedBytes(true);
Console.WriteLine($"Time={sw.Elapsed} Allocated={end - start} OutputSize={new FileInfo("Data.csv").Length}");
struct MyData
{
public Guid Id;
public DateTime TimeStamp;
public double LocX;
public double LocY;
}
sealed class MyDataReader : DbDataReader, IDbColumnSchemaGenerator
{
readonly MyData[] data;
readonly ReadOnlyCollection<DbColumn> schema;
int idx = -1;
public MyDataReader(MyData[] data)
{
this.data = data;
var cols = new DbColumn[]
{
new Col("Id", typeof(Guid)),
new Col("Timestamp", typeof(DateTime)),
new Col("LocX", typeof(double)),
new Col("LocY", typeof(double)),
};
this.schema = new ReadOnlyCollection<DbColumn>(cols);
}
class Col : DbColumn
{
public Col(string name, Type type)
{
this.ColumnName = name;
this.DataType = type;
this.AllowDBNull = false;
}
}
public override object this[int ordinal] => GetValue(ordinal);
public override object this[string name] => GetValue(GetOrdinal(name));
public override int Depth => 0;
public override int FieldCount => 4;
public override bool HasRows => true;
public override bool IsClosed => false;
public override int RecordsAffected => 0;
public override bool GetBoolean(int ordinal)
{
throw new NotImplementedException();
}
public override byte GetByte(int ordinal)
{
throw new NotImplementedException();
}
public override long GetBytes(int ordinal, long dataOffset, byte[]? buffer, int bufferOffset, int length)
{
throw new NotImplementedException();
}
public override char GetChar(int ordinal)
{
throw new NotImplementedException();
}
public override long GetChars(int ordinal, long dataOffset, char[]? buffer, int bufferOffset, int length)
{
throw new NotImplementedException();
}
public ReadOnlyCollection<DbColumn> GetColumnSchema()
{
return schema;
}
public override string GetDataTypeName(int ordinal)
{
throw new NotImplementedException();
}
public override DateTime GetDateTime(int ordinal)
{
if (ordinal == 1)
return data[idx].TimeStamp;
throw new ArgumentOutOfRangeException();
}
public override decimal GetDecimal(int ordinal)
{
throw new NotImplementedException();
}
public override double GetDouble(int ordinal)
{
switch (ordinal)
{
case 2:
return data[idx].LocX;
case 3:
return data[idx].LocY;
}
throw new ArgumentOutOfRangeException();
}
public override IEnumerator GetEnumerator()
{
throw new NotImplementedException();
}
public override Type GetFieldType(int ordinal)
{
return ordinal switch
{
0 => typeof(Guid),
1 => typeof(DateTime),
2 => typeof(double),
3 => typeof(double),
_ => throw new ArgumentOutOfRangeException()
};
}
public override float GetFloat(int ordinal)
{
throw new NotImplementedException();
}
public override Guid GetGuid(int ordinal)
{
if (ordinal == 0)
{
return data[idx].Id;
}
throw new ArgumentOutOfRangeException();
}
public override short GetInt16(int ordinal)
{
throw new NotImplementedException();
}
public override int GetInt32(int ordinal)
{
throw new NotImplementedException();
}
public override long GetInt64(int ordinal)
{
throw new NotImplementedException();
}
public override string GetName(int ordinal)
{
return schema[ordinal].ColumnName;
}
public override int GetOrdinal(string name)
{
throw new NotImplementedException();
}
public override string GetString(int ordinal)
{
throw new NotImplementedException();
}
public override object GetValue(int ordinal)
{
throw new NotImplementedException();
}
public override int GetValues(object[] values)
{
throw new NotImplementedException();
}
public override bool IsDBNull(int ordinal)
{
// You might have to worry about this if any of your members are nullable refs.
return false;
}
public override bool NextResult()
{
return false;
}
public override bool Read()
{
idx++;
return idx < data.Length;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment