json flattener and naivebayes inference
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
void Main() | |
{ | |
var exampleJson = @" | |
{ | |
""random"": 37, | |
""random float"": 51.886, | |
""bool"": true, | |
""date"": ""1982-05-19"", | |
""regEx"": ""hellooooooooo to you"", | |
""enum"": ""json"", | |
""firstname"": ""Henriette"", | |
""lastname"": ""Solitta"", | |
""city"": ""Melekeok"", | |
""country"": ""United States"", | |
""countryCode"": ""SD"", | |
""email uses current data"": ""Henriette.Solitta@gmail.com"", | |
""email from expression"": ""Henriette.Solitta@yopmail.com"", | |
""array"": [ | |
""Tani"", | |
""Tilly"", | |
""Blake"", | |
""Lusa"", | |
""Codie"" | |
], | |
""array of objects"": [ | |
{ | |
""index"": 0, | |
""index start at 5"": 5 | |
}, | |
{ | |
""index"": 1, | |
""index start at 5"": 6 | |
}, | |
{ | |
""index"": 2, | |
""index start at 5"": 7 | |
} | |
] | |
} | |
"; | |
new JsonFlattener().FlattenText(exampleJson).Dump(); | |
} | |
void NaiveBayes(List<List<string>> causes, List<List<string>> effects) | |
{ | |
Dictionary<T, int> countEvents<T>(IEnumerable<IEnumerable<T>> collection, IEnumerable<T> distinctEvents) | |
{ | |
return distinctEvents | |
.Select(e => (e, collection.Where(t => t.Contains(e)).Count())) | |
.ToDictionary(p => p.Item1, p => p.Item2); | |
} | |
var causeEvents = causes.SelectMany(x => x).Distinct(); | |
var effectEvents = effects.SelectMany(x => x).Distinct(); | |
var causeEventCount = countEvents(causes, causeEvents); | |
var effectEventCount = countEvents(effects, effectEvents); | |
var causeEventMargProb = causeEventCount | |
.ToDictionary(x => x.Key, x => Math.Log((double)x.Value) - Math.Log((double)causes.Count)); | |
var effectEventMargProb = effectEventCount | |
.ToDictionary(x => x.Key, x => Math.Log((double)x.Value) - Math.Log((double)effects.Count)); | |
var joinEvents = Enumerable.Zip(causes, effects) | |
.SelectMany( | |
x => ( | |
from a in x.First | |
from b in x.Second | |
select (a, b)) | |
); | |
var joinProb = ( | |
from c in causeEvents | |
from e in effectEvents | |
select new | |
{ | |
Events = new { Cause = c, Effect = e }, | |
Probability = Math.Log((double)joinEvents.Where(x => x == (c, e)).Count()) - Math.Log((double)joinEvents.Count()) | |
}); | |
var likelyEffects = causeEvents. | |
ToDictionary( | |
x => x, | |
x => joinProb | |
.Where(je => je.Events.Cause == x) | |
.Select(je => | |
(je.Events.Effect, | |
je.Probability - causeEventMargProb[je.Events.Cause])) | |
.ToList() | |
.OrderByDescending(x => x.Item2) | |
); | |
likelyEffects.Dump(); | |
} | |
public class JsonFlattener | |
{ | |
public List<string> FlattenText(string jsonStr) | |
{ | |
var token = JToken.Parse(jsonStr); | |
var result = new List<string>(); | |
FlattenRecursive(token, "_", result); | |
return result; | |
} | |
private void FlattenRecursive(JToken token, string namePrefix, List<string> resultList, bool includeValue=true, bool includeHasValue=true) | |
{ | |
if(token.Type == JTokenType.Array) | |
{ | |
int i = 0; | |
foreach(var child in token.Children()) | |
{ | |
var newPrefix = namePrefix + "[" + i.ToString() + "]"; | |
FlattenRecursive(child, newPrefix, resultList, includeValue, includeHasValue); | |
i++; | |
} | |
} | |
else if (token.Type == JTokenType.Object) | |
{ | |
JObject obj = (JObject)token; | |
foreach (var item in obj) | |
{ | |
var newPrefix = namePrefix + "." + item.Key; | |
FlattenRecursive(item.Value, newPrefix, resultList, includeValue, includeHasValue); | |
} | |
} | |
else | |
{ | |
if(includeHasValue) resultList.Add(namePrefix + "_HasValue"); | |
if(includeValue) resultList.Add(namePrefix + "=" + token.ToString()); | |
} | |
} | |
} | |
public interface IEventListor<T> | |
where T : IEquatable<T> | |
{ | |
IList<T> GetElements(); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment