Created
June 5, 2020 09:56
-
-
Save JoshuaManton/551cbdec63003d0930ef86ab6b0edafd to your computer and use it in GitHub Desktop.
A JSON serializer/deserializer.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package main | |
| import "core:fmt" | |
| import "core:os" | |
| import "core:mem" | |
| import "core:strings" | |
| import rt "core:runtime" | |
| import "core:strconv" | |
| main :: proc() { | |
| file_data, ok := os.read_entire_file("example.json"); | |
| assert(ok); | |
| lexer: Lexer; | |
| init_lexer(&lexer, cast(string)file_data); | |
| base_node := parse_value(&lexer); | |
| my_value: []Test_Value; | |
| write_value(base_node, &my_value); | |
| // pretty_print(my_value); | |
| serialized := serialize_to_json(&my_value); | |
| os.write_entire_file("serialized.json", transmute([]byte)serialized); | |
| // fmt.println(serialized); | |
| relexer: Lexer; | |
| init_lexer(&relexer, cast(string)serialized); | |
| redeserialized := parse_value(&relexer); | |
| my_value_redeserialized: []Test_Value; | |
| write_value(base_node, &my_value_redeserialized); | |
| pretty_print(my_value_redeserialized); | |
| } | |
| Test_Value :: struct { | |
| _id: string, | |
| index: int, | |
| guid: string, | |
| isActive: bool, | |
| balance: string, | |
| picture: string, | |
| age: int, | |
| eyeColor: string, | |
| name: string, | |
| gender: string, | |
| company: string, | |
| email: string, | |
| phone: string, | |
| address: string, | |
| about: string, | |
| registered: string, | |
| latitude: f32, | |
| longitude: f32, | |
| tags: []string, | |
| friends: []Friend, | |
| greeting: string, | |
| favoriteFruit: string, | |
| } | |
| Friend :: struct { | |
| id: int, | |
| name: string, | |
| } | |
| make_json_node :: proc(kind: $T) -> ^T { | |
| node := new(JSON_Node); | |
| node.kind = kind; | |
| return cast(^T)node; | |
| } | |
| parse_value :: proc(lexer: ^Lexer) -> ^JSON_Node { | |
| token, ok := peek(lexer); | |
| if !ok do return nil; | |
| switch token.type { | |
| case .Number: get_next_token(lexer); return cast(^JSON_Node)make_json_node(JSON_Number{token.int_value, token.float_value}); | |
| case .String: get_next_token(lexer); return cast(^JSON_Node)make_json_node(JSON_String{token.text}); // todo(josh): escape these strings? | |
| case .Left_Curly: return cast(^JSON_Node)parse_object(lexer); | |
| case .Left_Square: return cast(^JSON_Node)parse_array(lexer); | |
| case .True: get_next_token(lexer); return cast(^JSON_Node)make_json_node(JSON_True{}); | |
| case .False: get_next_token(lexer); return cast(^JSON_Node)make_json_node(JSON_False{}); | |
| case .Null: get_next_token(lexer); return cast(^JSON_Node)make_json_node(JSON_Null{}); | |
| case .Right_Curly: { | |
| unexpected_token(token); | |
| return nil; | |
| } | |
| case .Right_Square: { | |
| unexpected_token(token); | |
| return nil; | |
| } | |
| case .Colon: { | |
| unexpected_token(token); | |
| return nil; | |
| } | |
| case .Comma: { | |
| unexpected_token(token); | |
| return nil; | |
| } | |
| case .EOF: { | |
| unexpected_token(token); | |
| return nil; | |
| } | |
| case .None: { | |
| panic("wat"); | |
| } | |
| case: panic(tprint(token.type)); | |
| } | |
| unimplemented(); | |
| return nil; | |
| } | |
| parse_object :: proc(lexer: ^Lexer) -> ^JSON_Object { | |
| _, ok := expect(lexer, .Left_Curly); | |
| if !ok do return nil; | |
| fields: [dynamic]Field; | |
| for { | |
| token, ok := peek(lexer); | |
| if !ok do return nil; | |
| if token.type == .Right_Curly { | |
| get_next_token(lexer); | |
| return make_json_node(JSON_Object{fields[:]}); | |
| } | |
| if token.type != .String { | |
| unexpected_token(token); | |
| } | |
| field_name, ok2 := expect(lexer, .String); | |
| if !ok2 do return nil; | |
| _, ok3 := expect(lexer, .Colon); | |
| if !ok3 do return nil; | |
| value := parse_value(lexer); | |
| if value == nil do return nil; | |
| append(&fields, Field{field_name.text, value}); | |
| comma, ok4 := peek(lexer); | |
| if !ok4 do return nil; | |
| if comma.type == .Comma do get_next_token(lexer); | |
| } | |
| // todo(josh): if we get here that means end of text from within object | |
| unimplemented(); | |
| return nil; | |
| } | |
| parse_array :: proc(lexer: ^Lexer) -> ^JSON_Array { | |
| _, ok := expect(lexer, .Left_Square); | |
| if !ok do return nil; | |
| elements: [dynamic]^JSON_Node; | |
| for { | |
| token, ok := peek(lexer); | |
| if !ok do return nil; | |
| if token.type == .Right_Square { | |
| get_next_token(lexer); | |
| return make_json_node(JSON_Array{elements[:]}); | |
| } | |
| value := parse_value(lexer); | |
| if value == nil do return nil; | |
| append(&elements, value); | |
| comma, ok4 := peek(lexer); | |
| if !ok4 do return nil; | |
| if comma.type == .Comma do get_next_token(lexer); | |
| } | |
| // todo(josh): if we get here that means end of text from within object | |
| unimplemented(); | |
| return nil; | |
| } | |
| unexpected_token :: proc(token: Token) { | |
| error_message(token.location, tprint("Unexpected token: ", token.type)); | |
| } | |
| write_value :: proc(value: ^JSON_Node, dst: ^$Type) { | |
| write_value_ti(value, dst, type_info_of(Type)); | |
| } | |
| write_value_ti :: proc(value: ^JSON_Node, dst: rawptr, ti: ^rt.Type_Info) { | |
| switch kind in &value.kind { | |
| case JSON_Object: { | |
| struct_ti := unwrap_struct_ti(ti); | |
| for field in kind.fields { | |
| for name, idx in struct_ti.names { | |
| offset := struct_ti.offsets[idx]; | |
| type := struct_ti.types[idx]; | |
| if name == field.name { | |
| write_value_ti(field.value, mem.ptr_offset(cast(^byte)dst, cast(int)offset), type); | |
| } | |
| } | |
| } | |
| } | |
| case JSON_Array: { | |
| #partial | |
| switch ti_kind in ti.variant { | |
| case rt.Type_Info_Array: { | |
| array_length := min(len(kind.elements), ti_kind.count); | |
| for idx in 0..<array_length { | |
| array_dst := mem.ptr_offset(cast(^byte)dst, cast(int)idx * ti_kind.elem_size); | |
| write_value_ti(kind.elements[idx], array_dst, ti_kind.elem); | |
| } | |
| } | |
| case rt.Type_Info_Dynamic_Array: { | |
| memory := make([]byte, len(kind.elements) * ti_kind.elem_size); | |
| for idx in 0..<len(kind.elements) { | |
| array_dst := mem.ptr_offset(cast(^byte)&memory[0], idx * ti_kind.elem_size); | |
| write_value_ti(kind.elements[idx], array_dst, ti_kind.elem); | |
| } | |
| (cast(^mem.Raw_Dynamic_Array)dst)^ = mem.Raw_Dynamic_Array{&memory[0], len(kind.elements), len(kind.elements), {}}; // todo(josh): what do for allocator?? | |
| } | |
| case rt.Type_Info_Slice: { | |
| memory := make([]byte, len(kind.elements) * ti_kind.elem_size); | |
| for idx in 0..<len(kind.elements) { | |
| array_dst := mem.ptr_offset(cast(^byte)&memory[0], idx * ti_kind.elem_size); | |
| write_value_ti(kind.elements[idx], array_dst, ti_kind.elem); | |
| } | |
| (cast(^mem.Raw_Slice)dst)^ = mem.Raw_Slice{&memory[0], len(kind.elements)}; | |
| } | |
| case: panic(tprint(ti_kind)); | |
| } | |
| } | |
| case JSON_String: { | |
| str_ti := &ti.variant.(rt.Type_Info_String); | |
| if str_ti.is_cstring { | |
| (cast(^cstring)dst)^ = strings.clone_to_cstring(kind.value); | |
| } | |
| else { | |
| (cast(^string)dst)^ = strings.clone(kind.value); | |
| } | |
| } | |
| case JSON_Number: { | |
| #partial | |
| switch number_ti in ti.variant { | |
| case rt.Type_Info_Integer: { | |
| if number_ti.signed { | |
| switch number_ti.endianness { | |
| case .Platform: { | |
| switch ti.size { | |
| case 1: (cast(^i8 )dst)^ = cast(i8 )kind.int_value; | |
| case 2: (cast(^i16)dst)^ = cast(i16)kind.int_value; | |
| case 4: (cast(^i32)dst)^ = cast(i32)kind.int_value; | |
| case 8: (cast(^i64)dst)^ = cast(i64)kind.int_value; | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case .Little: { | |
| switch ti.size { | |
| case 2: (cast(^i16le)dst)^ = cast(i16le)kind.int_value; | |
| case 4: (cast(^i32le)dst)^ = cast(i32le)kind.int_value; | |
| case 8: (cast(^i64le)dst)^ = cast(i64le)kind.int_value; | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case .Big: { | |
| switch ti.size { | |
| case 2: (cast(^i16be)dst)^ = cast(i16be)kind.int_value; | |
| case 4: (cast(^i32be)dst)^ = cast(i32be)kind.int_value; | |
| case 8: (cast(^i64be)dst)^ = cast(i64be)kind.int_value; | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case: panic(tprint(number_ti.endianness)); | |
| } | |
| } | |
| else { | |
| switch number_ti.endianness { | |
| case .Platform: { | |
| switch ti.size { | |
| case 1: (cast(^u8 )dst)^ = cast(u8 )kind.int_value; | |
| case 2: (cast(^u16)dst)^ = cast(u16)kind.int_value; | |
| case 4: (cast(^u32)dst)^ = cast(u32)kind.int_value; | |
| case 8: (cast(^u64)dst)^ = cast(u64)kind.int_value; | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case .Little: { | |
| switch ti.size { | |
| case 2: (cast(^u16le)dst)^ = cast(u16le)kind.int_value; | |
| case 4: (cast(^u32le)dst)^ = cast(u32le)kind.int_value; | |
| case 8: (cast(^u64le)dst)^ = cast(u64le)kind.int_value; | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case .Big: { | |
| switch ti.size { | |
| case 2: (cast(^u16be)dst)^ = cast(u16be)kind.int_value; | |
| case 4: (cast(^u32be)dst)^ = cast(u32be)kind.int_value; | |
| case 8: (cast(^u64be)dst)^ = cast(u64be)kind.int_value; | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case: panic(tprint(number_ti.endianness)); | |
| } | |
| } | |
| } | |
| case rt.Type_Info_Float: { | |
| switch number_ti.endianness { | |
| case .Platform: { | |
| switch ti.size { | |
| case 4: (cast(^f32)dst)^ = cast(f32)kind.float_value; | |
| case 8: (cast(^f64)dst)^ = cast(f64)kind.float_value; | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case .Little: { | |
| switch ti.size { | |
| case 4: (cast(^f32le)dst)^ = cast(f32le)kind.float_value; | |
| case 8: (cast(^f64le)dst)^ = cast(f64le)kind.float_value; | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case .Big: { | |
| switch ti.size { | |
| case 4: (cast(^f32be)dst)^ = cast(f32be)kind.float_value; | |
| case 8: (cast(^f64be)dst)^ = cast(f64be)kind.float_value; | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case: panic(tprint(number_ti.endianness)); | |
| } | |
| } | |
| } | |
| } | |
| case JSON_True: { | |
| switch ti.size { | |
| case 1: (cast(^bool)dst)^ = true; | |
| case 2: (cast(^b16 )dst)^ = true; | |
| case 4: (cast(^b32 )dst)^ = true; | |
| case 8: (cast(^b64 )dst)^ = true; | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case JSON_False: { | |
| switch ti.size { | |
| case 1: (cast(^bool)dst)^ = false; | |
| case 2: (cast(^b16 )dst)^ = false; | |
| case 4: (cast(^b32 )dst)^ = false; | |
| case 8: (cast(^b64 )dst)^ = false; | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case JSON_Null: { | |
| (cast(^rawptr)dst)^ = nil; | |
| } | |
| case: panic(tprint(kind)); | |
| } | |
| } | |
| unwrap_struct_ti :: proc(ti: ^rt.Type_Info) -> ^rt.Type_Info_Struct { | |
| if named, ok := ti.variant.(rt.Type_Info_Named); ok { | |
| return &named.base.variant.(rt.Type_Info_Struct); | |
| } | |
| return &ti.variant.(rt.Type_Info_Struct); | |
| } | |
| serialize_to_json :: proc(thing: ^$Type) -> string { | |
| ti := type_info_of(Type); | |
| sb: strings.Builder; | |
| serialize_to_json_ti(&sb, thing, ti, 0); | |
| return strings.to_string(sb); | |
| } | |
| serialize_to_json_ti :: proc(sb: ^strings.Builder, ptr: rawptr, ti: ^rt.Type_Info, indent_level: int) { | |
| do_indent :: proc(sb: ^strings.Builder, indent_level: int) { | |
| for i in 0..<indent_level { | |
| sbprint(sb, " "); | |
| } | |
| } | |
| indent_level := indent_level; | |
| if ptr == nil { | |
| sbprint(sb, "null"); | |
| return; | |
| } | |
| switch ti_kind in ti.variant { | |
| case rt.Type_Info_Named: { | |
| serialize_to_json_ti(sb, ptr, ti_kind.base, indent_level); | |
| } | |
| case rt.Type_Info_Integer: { | |
| if ti_kind.signed { | |
| switch ti_kind.endianness { | |
| case .Platform: { | |
| switch ti.size { | |
| case 1: sbprint(sb, (cast(^i8 )ptr)^); | |
| case 2: sbprint(sb, (cast(^i16)ptr)^); | |
| case 4: sbprint(sb, (cast(^i32)ptr)^); | |
| case 8: sbprint(sb, (cast(^i64)ptr)^); | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case .Little: { | |
| switch ti.size { | |
| case 2: sbprint(sb, (cast(^i16le)ptr)^); | |
| case 4: sbprint(sb, (cast(^i32le)ptr)^); | |
| case 8: sbprint(sb, (cast(^i64le)ptr)^); | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case .Big: { | |
| switch ti.size { | |
| case 2: sbprint(sb, (cast(^i16be)ptr)^); | |
| case 4: sbprint(sb, (cast(^i32be)ptr)^); | |
| case 8: sbprint(sb, (cast(^i64be)ptr)^); | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case: panic(tprint(ti_kind.endianness)); | |
| } | |
| } | |
| else { | |
| switch ti_kind.endianness { | |
| case .Platform: { | |
| switch ti.size { | |
| case 1: sbprint(sb, (cast(^u8 )ptr)^); | |
| case 2: sbprint(sb, (cast(^u16)ptr)^); | |
| case 4: sbprint(sb, (cast(^u32)ptr)^); | |
| case 8: sbprint(sb, (cast(^u64)ptr)^); | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case .Little: { | |
| switch ti.size { | |
| case 2: sbprint(sb, (cast(^u16le)ptr)^); | |
| case 4: sbprint(sb, (cast(^u32le)ptr)^); | |
| case 8: sbprint(sb, (cast(^u64le)ptr)^); | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case .Big: { | |
| switch ti.size { | |
| case 2: sbprint(sb, (cast(^u16be)ptr)^); | |
| case 4: sbprint(sb, (cast(^u32be)ptr)^); | |
| case 8: sbprint(sb, (cast(^u64be)ptr)^); | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case: panic(tprint(ti_kind.endianness)); | |
| } | |
| } | |
| } | |
| case rt.Type_Info_Float: { | |
| switch ti_kind.endianness { | |
| case .Platform: { | |
| switch ti.size { | |
| case 4: sbprint(sb, (cast(^f32)ptr)^); | |
| case 8: sbprint(sb, (cast(^f64)ptr)^); | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case .Little: { | |
| switch ti.size { | |
| case 4: sbprint(sb, (cast(^f32le)ptr)^); | |
| case 8: sbprint(sb, (cast(^f64le)ptr)^); | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case .Big: { | |
| switch ti.size { | |
| case 4: sbprint(sb, (cast(^f32be)ptr)^); | |
| case 8: sbprint(sb, (cast(^f64be)ptr)^); | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case: panic(tprint(ti_kind.endianness)); | |
| } | |
| } | |
| case rt.Type_Info_Rune: { | |
| sbprint(sb, (cast(^rune)ptr)^); | |
| } | |
| case rt.Type_Info_String: { | |
| if ti_kind.is_cstring { | |
| sbprint(sb, '"', (cast(^cstring)ptr)^, '"'); | |
| } | |
| else { | |
| sbprint(sb, '"', (cast(^string)ptr)^, '"'); | |
| } | |
| } | |
| case rt.Type_Info_Boolean: { | |
| switch ti.size { | |
| case 1: sbprint(sb, (cast(^bool)ptr)^); | |
| case 2: sbprint(sb, (cast(^b16 )ptr)^); | |
| case 4: sbprint(sb, (cast(^b32 )ptr)^); | |
| case 8: sbprint(sb, (cast(^b64 )ptr)^); | |
| case: panic(tprint(ti.size)); | |
| } | |
| } | |
| case rt.Type_Info_Array: { | |
| sbprint(sb, "[\n"); | |
| indent_level += 1; | |
| for idx in 0..<ti_kind.count { | |
| do_indent(sb, indent_level); | |
| serialize_to_json_ti(sb, mem.ptr_offset(cast(^byte)ptr, idx * ti_kind.elem_size), ti_kind.elem, indent_level); | |
| if idx != ti_kind.count-1 { | |
| sbprint(sb, ","); | |
| } | |
| sbprint(sb, "\n"); | |
| } | |
| indent_level -= 1; | |
| do_indent(sb, indent_level); | |
| sbprint(sb, "]"); | |
| } | |
| case rt.Type_Info_Dynamic_Array: { | |
| sbprint(sb, "[\n"); | |
| indent_level += 1; | |
| raw := (cast(^mem.Raw_Dynamic_Array)ptr)^; | |
| for idx in 0..<raw.len { | |
| do_indent(sb, indent_level); | |
| serialize_to_json_ti(sb, mem.ptr_offset(cast(^byte)raw.data, idx * ti_kind.elem_size), ti_kind.elem, indent_level); | |
| if idx != raw.len-1 { | |
| sbprint(sb, ","); | |
| } | |
| sbprint(sb, "\n"); | |
| } | |
| indent_level -= 1; | |
| do_indent(sb, indent_level); | |
| sbprint(sb, "]"); | |
| } | |
| case rt.Type_Info_Slice: { | |
| sbprint(sb, "[\n"); | |
| indent_level += 1; | |
| raw := (cast(^mem.Raw_Slice)ptr)^; | |
| for idx in 0..<raw.len { | |
| do_indent(sb, indent_level); | |
| serialize_to_json_ti(sb, mem.ptr_offset(cast(^byte)raw.data, idx * ti_kind.elem_size), ti_kind.elem, indent_level); | |
| if idx != raw.len-1 { | |
| sbprint(sb, ","); | |
| } | |
| sbprint(sb, "\n"); | |
| } | |
| indent_level -= 1; | |
| do_indent(sb, indent_level); | |
| sbprint(sb, "]"); | |
| } | |
| case rt.Type_Info_Struct: { | |
| sbprint(sb, "{\n"); | |
| indent_level += 1; | |
| for name, idx in ti_kind.names { | |
| offset := ti_kind.offsets[idx]; | |
| type := ti_kind.types[idx]; | |
| do_indent(sb, indent_level); | |
| sbprint(sb, "\"", name, "\": "); | |
| serialize_to_json_ti(sb, mem.ptr_offset(cast(^byte)ptr, cast(int)offset), type, indent_level); | |
| if idx != len(ti_kind.names)-1 { | |
| sbprint(sb, ","); | |
| } | |
| sbprint(sb, "\n"); | |
| } | |
| indent_level -= 1; | |
| do_indent(sb, indent_level); | |
| sbprint(sb, "}"); | |
| } | |
| case rt.Type_Info_Union: unimplemented("Type_Info_Union"); | |
| case rt.Type_Info_Enum: unimplemented("Type_Info_Enum"); | |
| case rt.Type_Info_Map: unimplemented("Type_Info_Map"); | |
| case rt.Type_Info_Bit_Field: unimplemented("Type_Info_Bit_Field"); | |
| case rt.Type_Info_Bit_Set: unimplemented("Type_Info_Bit_Set"); | |
| case rt.Type_Info_Opaque: unimplemented("Type_Info_Opaque"); | |
| case rt.Type_Info_Simd_Vector: unimplemented("Type_Info_Simd_Vector"); | |
| case rt.Type_Info_Relative_Pointer: unimplemented("Type_Info_Relative_Pointer"); | |
| case rt.Type_Info_Relative_Slice: unimplemented("Type_Info_Relative_Slice"); | |
| case rt.Type_Info_Enumerated_Array: unimplemented("Type_Info_Enumerated_Array"); | |
| case rt.Type_Info_Tuple: unimplemented("Type_Info_Tuple"); | |
| case rt.Type_Info_Any: unimplemented("Type_Info_Any"); | |
| case rt.Type_Info_Type_Id: unimplemented("Type_Info_Type_Id"); | |
| case rt.Type_Info_Pointer: unimplemented("Type_Info_Pointer"); | |
| case rt.Type_Info_Procedure: unimplemented("Type_Info_Procedure"); | |
| case rt.Type_Info_Complex: unimplemented("Type_Info_Complex"); | |
| case rt.Type_Info_Quaternion: unimplemented("Type_Info_Quaternion"); | |
| } | |
| } | |
| JSON_Node :: struct { | |
| kind: union { // note(josh): this union must be at the top because we cast pointers | |
| JSON_Object, | |
| JSON_Array, | |
| JSON_String, | |
| JSON_Number, | |
| JSON_True, | |
| JSON_False, | |
| JSON_Null, | |
| }, | |
| } | |
| JSON_Object :: struct { | |
| fields: []Field, | |
| } | |
| Field :: struct { | |
| name: string, | |
| value: ^JSON_Node, | |
| } | |
| JSON_Array :: struct { | |
| elements: []^JSON_Node, | |
| } | |
| JSON_String :: struct { | |
| value: string, | |
| } | |
| JSON_Number :: struct { | |
| int_value: i64, | |
| float_value: f64, | |
| } | |
| JSON_True :: struct { | |
| } | |
| JSON_False :: struct { | |
| } | |
| JSON_Null :: struct { | |
| } | |
| Lexer :: struct { | |
| text: string, | |
| cur_idx: int, | |
| location: Location, | |
| } | |
| Token :: struct { | |
| text: string, | |
| int_value: i64, | |
| float_value: f64, | |
| type: Token_Type, | |
| location: Location, | |
| } | |
| Location :: struct { | |
| line: int, | |
| char: int, | |
| } | |
| Token_Type :: enum { | |
| None, | |
| Number, | |
| String, | |
| Comma, | |
| Colon, | |
| Left_Curly, | |
| Right_Curly, | |
| Left_Square, | |
| Right_Square, | |
| True, | |
| False, | |
| Null, | |
| EOF, | |
| } | |
| init_lexer :: proc(lexer: ^Lexer, text: string) { | |
| lexer.text = text; | |
| lexer.location.line = 1; | |
| lexer.location.char = 1; | |
| } | |
| get_next_token :: proc(lexer: ^Lexer) -> (Token, bool) { // todo(josh): error values? | |
| if lexer.cur_idx >= len(lexer.text) { | |
| return Token{"", 0, 0, .EOF, {}}, true; | |
| } | |
| eat_whitespace(lexer); | |
| current_location := lexer.location; | |
| c := lexer.text[lexer.cur_idx]; | |
| switch c { | |
| case '"': { | |
| text, ok := scan_string(lexer); | |
| if !ok do return {}, false; | |
| token := Token{text, 0, 0, .String, current_location}; | |
| return token, true; | |
| } | |
| case '-', '0'..'9': { | |
| text, int_val, float_val, ok := scan_number(lexer); | |
| if !ok do return {}, false; | |
| token := Token{text, int_val, float_val, .Number, current_location}; | |
| return token, true; | |
| } | |
| case 'a'..'z': { | |
| text := scan_identifier(lexer); | |
| switch text { | |
| case "true": return Token{text, 0, 0, .True, current_location}, true; | |
| case "false": return Token{text, 0, 0, .False, current_location}, true; | |
| case "null": return Token{text, 0, 0, .Null, current_location}, true; | |
| } | |
| unreachable(); | |
| return {}, false; | |
| } | |
| case ',': advance(lexer, true); return Token{",", 0, 0, .Comma, current_location}, true; | |
| case ':': advance(lexer, true); return Token{":", 0, 0, .Colon, current_location}, true; | |
| case '{': advance(lexer, true); return Token{"{", 0, 0, .Left_Curly, current_location}, true; | |
| case '}': advance(lexer, true); return Token{"}", 0, 0, .Right_Curly, current_location}, true; | |
| case '[': advance(lexer, true); return Token{"[", 0, 0, .Left_Square, current_location}, true; | |
| case ']': advance(lexer, true); return Token{"]", 0, 0, .Right_Square, current_location}, true; | |
| case: panic(tprint(c)); | |
| } | |
| unreachable(); | |
| return {}, false; | |
| } | |
| peek :: proc(lexer: ^Lexer) -> (Token, bool) { | |
| lexer_copy := lexer^; | |
| token, ok := get_next_token(&lexer_copy); | |
| return token, ok; | |
| } | |
| expect :: proc(lexer: ^Lexer, type: Token_Type) -> (Token, bool) { | |
| token, ok := get_next_token(lexer); | |
| if !ok do return {}, false; | |
| if token.type == type { | |
| return token, true; | |
| } | |
| unexpected_token(token); // todo(josh): print what we DID expect | |
| return token, false; | |
| } | |
| scan_string :: proc(lexer: ^Lexer) -> (string, bool) { | |
| assert(lexer.text[lexer.cur_idx] == '"'); | |
| advance(lexer, true); | |
| start := lexer.cur_idx; | |
| // todo(josh): handle nested strings | |
| for lexer.cur_idx < len(lexer.text) && lexer.text[lexer.cur_idx] != '"' { | |
| advance(lexer, true); | |
| } | |
| if lexer.cur_idx >= len(lexer.text) { | |
| error_message(lexer.location, "End of text from within string."); | |
| return {}, false; | |
| } | |
| assert(lexer.text[lexer.cur_idx] == '"'); | |
| end := lexer.cur_idx; | |
| advance(lexer, true); | |
| str := lexer.text[start:end]; | |
| return str, true; | |
| } | |
| scan_number :: proc(lexer: ^Lexer) -> (string, i64, f64, bool) { | |
| start := lexer.cur_idx; | |
| if lexer.cur_idx < len(lexer.text) && lexer.text[lexer.cur_idx] == '-' { | |
| advance(lexer, false); | |
| } | |
| if lexer.cur_idx < len(lexer.text) && lexer.text[lexer.cur_idx] == '0' { | |
| advance(lexer, false); | |
| } | |
| else { | |
| num_loop_1: for lexer.cur_idx < len(lexer.text) { | |
| switch lexer.text[lexer.cur_idx] { | |
| case '0'..'9': advance(lexer, false); | |
| case: break num_loop_1; | |
| } | |
| } | |
| } | |
| int_end := lexer.cur_idx; | |
| if lexer.cur_idx < len(lexer.text) && lexer.text[lexer.cur_idx] == '.' { | |
| advance(lexer, false); | |
| num_loop_2: for lexer.cur_idx < len(lexer.text) { | |
| switch lexer.text[lexer.cur_idx] { | |
| case '0'..'9': advance(lexer, false); | |
| case: break num_loop_2; | |
| } | |
| } | |
| } | |
| end := lexer.cur_idx; | |
| // todo(josh): scientific notation | |
| text := lexer.text[start:end]; | |
| if text == "-" { | |
| // todo(josh): this error message is printing a character number one less than what I would expect given | |
| // the length of my input when testing it. Investigate. | |
| error_message(lexer.location, "Expected number after minus sign."); | |
| return "", 0, 0, false; | |
| } | |
| int_text := lexer.text[start:int_end]; | |
| int_value, ok1 := strconv.parse_i64(int_text); assert(ok1); | |
| float_value, ok2 := strconv.parse_f64(text); assert(ok2); | |
| return text, int_value, float_value, true; | |
| } | |
| scan_identifier :: proc(lexer: ^Lexer) -> string { | |
| start := lexer.cur_idx; | |
| ident_loop: for lexer.cur_idx < len(lexer.text) { | |
| switch lexer.text[lexer.cur_idx] { | |
| case 'a'..'z': advance(lexer, false); | |
| case: break ident_loop; | |
| } | |
| } | |
| end := lexer.cur_idx; | |
| text := lexer.text[start:end]; | |
| return text; | |
| } | |
| advance :: proc(lexer: ^Lexer, do_eat_whitespace: bool) { | |
| lexer.cur_idx += 1; | |
| lexer.location.char += 1; | |
| if do_eat_whitespace { | |
| eat_whitespace(lexer); | |
| } | |
| } | |
| eat_whitespace :: proc(lexer: ^Lexer) { | |
| whitespace_loop: for lexer.cur_idx < len(lexer.text) { | |
| switch lexer.text[lexer.cur_idx] { | |
| case '\n': { | |
| lexer.cur_idx += 1; | |
| lexer.location.line += 1; | |
| lexer.location.char = 1; | |
| } | |
| case ' ', '\t', '\r': { | |
| lexer.cur_idx += 1; | |
| lexer.location.char += 1; | |
| } | |
| case: break whitespace_loop; | |
| } | |
| } | |
| } | |
| error_message :: proc(location: Location, message: string) { | |
| fmt.printf("Error at %d:%d: %s\n", location.line, location.char, message); | |
| } | |
| tprint :: fmt.tprint; | |
| println :: fmt.println; | |
| sbprint :: fmt.sbprint; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment