Skip to content

Instantly share code, notes, and snippets.

@taddison
Created August 17, 2018 12:58
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save taddison/13f0af7312c728b46efd0f138fd8a365 to your computer and use it in GitHub Desktop.
Save taddison/13f0af7312c728b46efd0f138fd8a365 to your computer and use it in GitHub Desktop.
Read Avro EventHubs capture file (credit @Kadajski)
using Microsoft.Hadoop.Avro.Container;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
namespace AvroReader
{
public class AvroEventData
{
public AvroEventData(dynamic record)
{
SequenceNumber = (long)record.SequenceNumber;
Offset = (string)record.Offset;
DateTime.TryParse((string)record.EnqueuedTimeUtc, out var enqueuedTimeUtc);
EnqueuedTimeUtc = enqueuedTimeUtc;
SystemProperties = (Dictionary<string, object>)record.SystemProperties;
Properties = (Dictionary<string, object>)record.Properties;
Body = (byte[])record.Body;
}
public long SequenceNumber { get; set; }
public string Offset { get; set; }
public DateTime EnqueuedTimeUtc { get; set; }
public Dictionary<string, object> SystemProperties { get; set; }
public Dictionary<string, object> Properties { get; set; }
public byte[] Body { get; set; }
}
public static class AvroReader
{
public static void ReadRecursive<T>(string path, List<T> items) where T : new()
{
var files = Directory.GetFiles(path);
foreach (var file in files)
{
using (var stream = File.OpenRead(file))
using (var reader = AvroContainer.CreateGenericReader(stream))
{
try
{
while (reader.MoveNext())
{
foreach (dynamic result in reader.Current.Objects)
{
var record = new AvroEventData(result);
if (record.Body.Length == 0)
{
continue;
}
var obj = JsonConvert.DeserializeObject<T>(Encoding.UTF8.GetString(record.Body));
if (EqualityComparer<T>.Default.Equals(obj, default(T)) == false)
{
items.Add(obj);
}
}
}
}
catch (System.ArgumentOutOfRangeException)
{
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment