Skip to content

Instantly share code, notes, and snippets.

@leandromoh
Created December 24, 2023 02:37
Show Gist options
  • Save leandromoh/ed4a2d30ea43da39c3df56c6fa0d7d74 to your computer and use it in GitHub Desktop.
Save leandromoh/ed4a2d30ea43da39c3df56c6fa0d7d74 to your computer and use it in GitHub Desktop.
serialize objects to json then get md5 hashes without too much string allocations
using System.Buffers;
using System.Reflection;
using System.Text.Json;
using System.Text.Json.Serialization.Metadata;
public interface IHashGenerator
{
void GenerateHashes<T>(IEnumerable<T> source, Action<T, string> action);
void GenerateHashes<T, TKey>(IEnumerable<T> source, Func<T, TKey> selector, Action<T, string> action);
}
public class HashGenerator : IHashGenerator
{
private int _size;
public HashGenerator() : this((int)Math.Pow(2, 13)) { }
public HashGenerator(int bufferSize)
{
if (bufferSize < 1)
throw new ArgumentOutOfRangeException(nameof(bufferSize), $"{nameof(bufferSize)} must be greater than zero.");
_size = bufferSize;
}
public void GenerateHashes<T>(IEnumerable<T> source, Action<T, string> action)
{
GenerateHashes(source, x => x, action);
}
public void GenerateHashes<T, TKey>(IEnumerable<T> source, Func<T, TKey> selector, Action<T, string> action)
{
var pool = (byte[])null;
var mem = (MemoryStream)null;
var writer = (Utf8JsonWriter)null;
Init();
var options = new JsonSerializerOptions
{
TypeInfoResolver = new DefaultJsonTypeInfoResolver
{
Modifiers = { JsonExtensions.AlphabetizeProperties() },
},
};
try
{
foreach (var item in source)
{
var attempts = 0;
reloop:
//if (attempts++ > 15)
// throw new Exception($"Double buffer size {15}x was not enough. Abort with size of {_size}.");
try
{
var key = selector(item);
System.Text.Json.JsonSerializer.Serialize(writer, key, inputType: key.GetType(), options);
var md5 = pool.ToMd5(0, (int)mem.Position);
action(item, md5);
writer.Reset();
mem.Position = 0;
}
catch (NotSupportedException ex)
when (ex.Message == "Memory stream is not expandable.")
{
Reset();
goto reloop;
}
catch (TargetInvocationException ex)
when (ex.InnerException is System.Text.Json.JsonException js && js.Message == "The object or value could not be serialized.Path: $.")
{
Reset();
goto reloop;
}
}
}
finally
{
Dispose();
}
void Init()
{
pool = ArrayPool<byte>.Shared.Rent(_size);
mem = new MemoryStream(pool, writable: true);
writer = new Utf8JsonWriter(mem);
}
void Reset()
{
Dispose();
_size *= 2;
Init();
}
void Dispose()
{
ArrayPool<byte>.Shared.Return(pool);
mem.Dispose();
writer.Dispose();
}
}
}
public static class JsonExtensions
{
public static Action<JsonTypeInfo> AlphabetizeProperties(Type type)
{
return typeInfo =>
{
if (typeInfo.Kind != JsonTypeInfoKind.Object || !type.IsAssignableFrom(typeInfo.Type))
return;
AlphabetizeProperties()(typeInfo);
};
}
// https://stackoverflow.com/a/72593993
public static Action<JsonTypeInfo> AlphabetizeProperties()
{
return static typeInfo =>
{
if (typeInfo.Kind != JsonTypeInfoKind.Object)
return;
var properties = typeInfo.Properties.OrderBy(p => p.Name, StringComparer.Ordinal).ToList();
typeInfo.Properties.Clear();
for (int i = 0; i < properties.Count; i++)
{
properties[i].Order = i;
typeInfo.Properties.Add(properties[i]);
}
};
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment