Skip to content

Instantly share code, notes, and snippets.

@mrange

mrange/schema_0_example.cs

Last active June 5, 2021 21:01
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save mrange/53323f90b480f4496f347fad0125d866 to your computer and use it in GitHub Desktop.
Save mrange/53323f90b480f4496f347fad0125d866 to your computer and use it in GitHub Desktop.
Download ZIP
Schema inferrer F#
Raw
schema_0_example.cs
#nullable enable
namespace Responses.Customers
{
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;
/* Created model classes from input JSON:
{
"customers" : {
"totalCount" : 3,
"items" : [
null,
{
"customerId" : 3,
"name" : "Hello there",
"age" : 24,
"lastChanged" : "2021-05-05T13:42:00Z"
},
{
"customerId" : 4,
"name" : "Hello there",
"age" : null
},
null
]
}
}
*/
// ---------------------------------------------------------------------------
partial record Customers
{
[JsonPropertyName(@"items")]
public List<Customers_Items?>? Items { get; set; }
[JsonPropertyName(@"totalCount")]
public long? TotalCount { get; set; }
}
// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
partial record Customers_Items
{
[JsonPropertyName(@"age")]
public long? Age { get; set; }
[JsonPropertyName(@"customerId")]
public long? CustomerId { get; set; }
[JsonPropertyName(@"lastChanged")]
public DateTime? LastChanged { get; set; }
[JsonPropertyName(@"name")]
public string? Name { get; set; }
}
// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
partial record Response
{
[JsonPropertyName(@"customers")]
public Customers? Customers { get; set; }
}
// ---------------------------------------------------------------------------
}
namespace Responses.Customers2
{
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;
/* Created model classes from input JSON:
{
"data": {
"customers": {
"items": [
{
"customerId": 5,
"customerNumber": 10062,
"name": "Demo",
"sys_Deactivated": false,
"sys_RowVersion": 13430229,
"sys_UpdatedDateUTC": "2021-06-04T13:31:56Z",
"externalSystemCustomers": {
"items": [
{
"enum_SyncStatusId": "SaveCompleted",
"externalId": "11994050",
"externalSystemId": 5,
"sys_RowVersion": 13430231,
"sys_UpdatedDateUTC": "2021-06-04T13:31:56Z"
}
]
}
},
{
"customerId": 7,
"customerNumber": 10063,
"name": "Test",
"sys_Deactivated": false,
"sys_RowVersion": 13430230,
"sys_UpdatedDateUTC": "2021-06-04T13:31:56Z",
"externalSystemCustomers": {
"items": [
{
"enum_SyncStatusId": "SaveCompleted",
"externalId": "11994051",
"externalSystemId": 5,
"sys_RowVersion": 13430232,
"sys_UpdatedDateUTC": "2021-06-04T13:31:56Z"
}
]
}
}
]
}
}
}
*/
// ---------------------------------------------------------------------------
partial record Customers
{
[JsonPropertyName(@"items")]
public List<Customers_Items?>? Items { get; set; }
}
// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
partial record Customers_Items
{
[JsonPropertyName(@"customerId")]
public long? CustomerId { get; set; }
[JsonPropertyName(@"customerNumber")]
public long? CustomerNumber { get; set; }
[JsonPropertyName(@"externalSystemCustomers")]
public ExternalSystemCustomers? ExternalSystemCustomers { get; set; }
[JsonPropertyName(@"name")]
public string? Name { get; set; }
[JsonPropertyName(@"sys_Deactivated")]
public bool? Sys_Deactivated { get; set; }
[JsonPropertyName(@"sys_RowVersion")]
public long? Sys_RowVersion { get; set; }
[JsonPropertyName(@"sys_UpdatedDateUTC")]
public DateTime? Sys_UpdatedDateUTC { get; set; }
}
// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
partial record Data
{
[JsonPropertyName(@"customers")]
public Customers? Customers { get; set; }
}
// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
partial record ExternalSystemCustomers
{
[JsonPropertyName(@"items")]
public List<ExternalSystemCustomers_Items?>? Items { get; set; }
}
// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
partial record ExternalSystemCustomers_Items
{
[JsonPropertyName(@"enum_SyncStatusId")]
public string? Enum_SyncStatusId { get; set; }
[JsonPropertyName(@"externalId")]
public string? ExternalId { get; set; }
[JsonPropertyName(@"externalSystemId")]
public long? ExternalSystemId { get; set; }
[JsonPropertyName(@"sys_RowVersion")]
public long? Sys_RowVersion { get; set; }
[JsonPropertyName(@"sys_UpdatedDateUTC")]
public DateTime? Sys_UpdatedDateUTC { get; set; }
}
// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
partial record Response
{
[JsonPropertyName(@"data")]
public Data? Data { get; set; }
}
// ---------------------------------------------------------------------------
}
Raw
schema_1_inferrer.fs
namespace FsJsonSchema
open Newtonsoft.Json
open Newtonsoft.Json.Linq
open System
open System.Globalization
open System.IO
open System.Text
open System.Collections.Generic
module Details =
type JsonPathElement =
| MemberPath of string
| ArrayPath of int
open Details
type SchemaPathElement =
| SchemaMember of string
| SchemaArray
| SchemaNullable
type InferredSchema =
| Nothing
| Boolean
| Integer
| Decimal
| DateTime
| String
| Nullable of InferredSchema
| Array of InferredSchema
| Class of Map<string, InferredSchema>
member x.Name =
match x with
| Nothing -> "Nothing"
| Boolean -> "Boolean"
| Integer -> "Integer"
| Decimal -> "Decimal"
| DateTime -> "DateTime"
| String -> "String"
| Nullable y -> y.Name
| Array _ -> "Array"
| Class _ -> "Class"
type MemberDefinition =
{
IsCollectionNullable : bool
IsCollection : bool
IsNullable : bool
Type : string
Name : string
}
type ClassDefinition =
{
Name : string
Members : MemberDefinition array
}
module SchemaInferrer =
let inferSchema (jt : JToken) : InferredSchema =
let pathToString (path : JsonPathElement list) =
let sb = StringBuilder 32
let rec loop path =
match path with
| h::t ->
loop t
match h with
| MemberPath nm -> sb.Append('.').Append(nm) |> ignore
| ArrayPath i -> sb.Append('[').Append(i).Append(']') |> ignore
| _ ->
sb.Append '$' |> ignore
loop path
sb.ToString ()
let typeName o = o.GetType().Name
let incompatibleTypes path (s : InferredSchema) (jt : JToken) =
let lineInfo = jt :> IJsonLineInfo
failwithf
"Can't merge types : %s (Previously inferred) and %A (Json type at line:%d(%d), path:%s)"
s.Name
jt.Type
lineInfo.LineNumber
lineInfo.LinePosition
(pathToString path)
let inferValue path (s : InferredSchema) (jv : JValue) =
let likeInt =
match jv.Type with
| JTokenType.Integer -> true
| _ -> false
let likeDecimal =
match jv.Type with
| JTokenType.Float -> true
| _ -> false
let likeBoolean =
match jv.Type with
| JTokenType.Boolean -> true
| _ -> false
let likeString =
match jv.Type with
| JTokenType.String
| JTokenType.Date
| JTokenType.Guid
| JTokenType.Uri
| JTokenType.TimeSpan -> true
| _ -> false
let likeDateTime =
match jv.Type with
| JTokenType.Date -> true
| _ -> false
let s =
match s with
| Nothing | Boolean when likeBoolean -> Boolean
| Nothing | Integer when likeInt -> Integer
| Nothing | Integer | Decimal when likeDecimal -> Decimal
| Nothing | DateTime when likeDateTime -> DateTime
| Nothing | DateTime | String when likeString -> String
| _ ->
incompatibleTypes path s jv
s
let rec inferClass path (s : InferredSchema) (jo : JObject) =
let ms =
match s with
| Nothing -> Map.empty
| Class ms -> ms
| _ ->
incompatibleTypes path s jo
let jps = jo.Properties() |> Seq.toList
let rec loop ms (jps : JProperty list) =
match jps with
| jp::t ->
let s =
match ms |> Map.tryFind jp.Name with
| Some s -> s
| _ -> Nothing
let jt = jo.[jp.Name]
let path = (MemberPath jp.Name)::path
let s = infer path s jt
let ms = ms |> Map.add jp.Name s
loop ms t
| _ -> ms
let ms = loop ms jps
Class ms
and inferArray path (s : InferredSchema) (ja : JArray) =
let s =
match s with
| Nothing -> Nothing
| Array s -> s
| _ ->
incompatibleTypes path s ja
let c = ja.Count
let rec loop s i =
if i < c then
let jt = ja.[i]
let path = (ArrayPath i)::path
let s = infer path s jt
loop s (i + 1)
else
s
let s = loop s 0
Array s
and infer path s (jt : JToken) =
match jt.Type with
| JTokenType.Undefined
| JTokenType.Null ->
match s with
| Nullable _ -> s
| _ -> Nullable s
| _ ->
let s, sc =
match s with
| Nullable s -> s, Nullable
| _ -> s, id
let s =
match jt with
| (:? JValue as jv) -> inferValue path s jv
| (:? JObject as jo) -> inferClass path s jo
| (:? JArray as ja) -> inferArray path s ja
| _ ->
failwithf "Unexpected JToken @ %s : %A" (pathToString path) (typeName jt)
sc s
let s = infer [] Nothing jt
s
let inferSchemaFromJson json =
use sr = new StringReader(json)
use jr = new JsonTextReader(sr)
jr.Culture <- CultureInfo.InvariantCulture
jr.FloatParseHandling <- FloatParseHandling.Decimal
jr.DateParseHandling <- DateParseHandling.DateTime
jr.DateTimeZoneHandling <- DateTimeZoneHandling.RoundtripKind
jr.DateFormatString <- "yyyy-MM-ddTHH:mm:ssZ"
// let b, v = DateTime.TryParseExact(text, jr.DateFormatString, culture, DateTimeStyles.RoundtripKind)
let settings = JsonLoadSettings ()
settings.CommentHandling <- CommentHandling.Ignore
settings.DuplicatePropertyNameHandling <- DuplicatePropertyNameHandling.Error
settings.LineInfoHandling <- LineInfoHandling.Load
let jt = JToken.Load(jr, settings)
inferSchema jt
let flattenSchema (nameResolver : SchemaPathElement list -> string) (s : InferredSchema) : ClassDefinition list =
let rec flatten path cds cn c n nm s =
let md t = { IsCollectionNullable = cn; IsCollection = c; IsNullable = n; Type = t ; Name = nm }
match s with
| Nothing -> cds, md "object"
| Boolean -> cds, md "bool"
| Integer -> cds, md "long"
| Decimal -> cds, md "decimal"
| DateTime -> cds, md "DateTime"
| String -> cds, md "string"
| Nullable s -> flatten (SchemaNullable::path) cds cn c true nm s
| Array s -> flatten (SchemaArray::path) cds c true false nm s
| Class ms ->
let name = nameResolver path
let folder (cds, (mds: MemberDefinition list)) (kv : KeyValuePair<string, InferredSchema>) =
let nm = kv.Key
let cds, imd = flatten (SchemaMember nm::path) cds false false false nm kv.Value
let mds = imd::mds
cds, mds
let z = cds, List.empty
let cds, mds = ms |> Seq.fold folder z
{ Name = name; Members = mds |> List.toArray }::cds, md name
let cds, _ = flatten List.empty List.empty false false false "<UNDEFINED>" s
cds
module CsSchemaInferrer =
let InferFlatSchemaFromJson (nameResolver : Func<string seq, string>) (json : string) : ClassDefinition array =
let s = SchemaInferrer.inferSchemaFromJson json
let nr path =
let mapper pp =
match pp with
| SchemaMember m -> m
| SchemaNullable -> "@Nullable"
| SchemaArray -> "@Collection"
let p = path |> List.map mapper
nameResolver.Invoke p
SchemaInferrer.flattenSchema nr s |> List.toArray
Raw
schema_2_creator.tt
<#@ assembly name="$(SolutionDir)\FsJsonSchema\bin\Debug\net472\FsJsonSchema.dll" #>
<#@ assembly name="System.Core" #>
<#@ import namespace="System" #>
<#@ import namespace="System.Collections.Generic" #>
<#@ import namespace="System.Linq" #>
#nullable enable
<#
const string sampleJson = @"
{
""customers"" : {
""totalCount"" : 3,
""items"" : [
null,
{
""customerId"" : 3,
""name"" : ""Hello there"",
""age"" : 24,
""lastChanged"" : ""2021-05-05T13:42:00Z""
},
{
""customerId"" : 4,
""name"" : ""Hello there"",
""age"" : null
},
null
]
}
}";
const string sampleJson2 = @"{
""data"": {
""customers"": {
""items"": [
{
""customerId"": 5,
""customerNumber"": 10062,
""name"": ""Demo"",
""sys_Deactivated"": false,
""sys_RowVersion"": 13430229,
""sys_UpdatedDateUTC"": ""2021-06-04T13:31:56Z"",
""externalSystemCustomers"": {
""items"": [
{
""enum_SyncStatusId"": ""SaveCompleted"",
""externalId"": ""11994050"",
""externalSystemId"": 5,
""sys_RowVersion"": 13430231,
""sys_UpdatedDateUTC"": ""2021-06-04T13:31:56Z""
}
]
}
},
{
""customerId"": 7,
""customerNumber"": 10063,
""name"": ""Test"",
""sys_Deactivated"": false,
""sys_RowVersion"": 13430230,
""sys_UpdatedDateUTC"": ""2021-06-04T13:31:56Z"",
""externalSystemCustomers"": {
""items"": [
{
""enum_SyncStatusId"": ""SaveCompleted"",
""externalId"": ""11994051"",
""externalSystemId"": 5,
""sys_RowVersion"": 13430232,
""sys_UpdatedDateUTC"": ""2021-06-04T13:31:56Z""
}
]
}
}
]
}
}
}";
var rightPadType = 40;
var rightPadName = 30;
var model = new [] {
("Responses.Customers" , sampleJson ),
("Responses.Customers2" , sampleJson2 ),
};
foreach (var (ns, json) in model) {
var cds = FsJsonSchema.CsSchemaInferrer.InferFlatSchemaFromJson(NameResolver, json);
#>
namespace <#=ns#>
{
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;
/* Created model classes from input JSON:
<#=json#>
*/
<#
foreach (var cd in cds.OrderBy(cd => cd.Name)) {
#>
// ---------------------------------------------------------------------------
partial record <#=cd.Name#>
{
<#
foreach (var md in cd.Members.OrderBy(md => md.Name)) {
var type = md.Type;
type = type + "?";
if (md.IsCollection) {
type = "List<" + type + ">?";
}
#>
[JsonPropertyName(@"<#=md.Name#>")]
public <#=RightPad(type, rightPadType)#> <#=RightPad(UpperCase(md.Name), rightPadName)#> { get; set; }
<# } #>
}
// ---------------------------------------------------------------------------
<# } #>
}
<# } #>
<#+
static string RightPad(string s, int n) {
if (s.Length < n) {
return s + new string(' ', n - s.Length);
} else {
return s;
}
}
static string UpperCase(string s) {
if (s.Length > 0) {
var f = Char.ToUpperInvariant(s[0]);
return f + s.Substring(1);
} else {
return "";
}
}
static string NameResolver(IEnumerable<string> path) {
var isItems = false;
foreach (var p in path) {
switch(p) {
case "@Nullable":
break;
case "@Collection":
break;
case "items":
isItems = true;
break;
default:
return UpperCase(p) + (isItems ? "_Items" : "");
}
}
return "Response";
}
#>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment