README.md

Reading and writing JSON with GO

Problem: I want to read and serve JSON, but the JSON structure differs by query. I solve this with an interface FleetVehicle, a base Vehicle struct, and additional specialized structs that allow for differetnail parsing. The Fleet struct holds raw json in RawVehicles, and parsed objects/structs in Vehicles. By writing MarsahJSON and UnmarshalJSON for the Fleet struct, and I control how each JSON block is parsed into a different type of struct/vehicle. To run: GOPATH=${PWD} go run helloVariant.go, and check http://localhost:8080/world. This loads multiple JSON files into one Fleet object allowing me to serve a concatenation of the JSON files. For now, each JSON block must use the data field to hold a list of vehicles.

• helloVariant.go is the main file
• src/varparse/varparse.go holds structs for variants in JSON
• testVariant1.json and testVariant2.json are example files used by helloVariant.go
• hello.go is my initial attempt
• fleet.go, fleet_test.go, fleet_test_output.txt are examples of using JSON with different structures. These are carried over from the initial example, but no longer needed.

fleet.go

type FleetVehicle interface {
	MakeAndModel() string
}

type Vehicle struct {
	Type  string `json:"type"`
	Make  string `json:"make"`
	Model string `json:"model"`
}

func (v Vehicle) MakeAndModel() string {
	return fmt.Sprintf("%s %s", v.Make, v.Model)
}

type Car struct {
	Vehicle
	SeatingCapacity int     `json:"seatingCapacity"`
	TopSpeed        float64 `json:"topSpeed"`
}

type Truck struct {
	Vehicle
	PayloadCapacity float64 `json:"payloadCapacity"`
}

type Bike struct {
	Vehicle
	TopSpeed float64 `json:"topSpeed"`
}

type Fleet struct {
	Vehicles    []FleetVehicle    `json:"-"`
	RawVehicles []json.RawMessage `json:"vehicles"`
}

func (f *Fleet) MarshalJSON() ([]byte, error) {

	type fleet Fleet

	if f.Vehicles != nil {
		for _, v := range f.Vehicles {
			b, err := json.Marshal(v)
			if err != nil {
				return nil, err
			}
			f.RawVehicles = append(f.RawVehicles, b)
		}
	}

	return json.Marshal((*fleet)(f))
}

func (f *Fleet) UnmarshalJSON(b []byte) error {

	type fleet Fleet

	err := json.Unmarshal(b, (*fleet)(f))
	if err != nil {
		return err
	}

	for _, raw := range f.RawVehicles {

		var v Vehicle
		err = json.Unmarshal(raw, &v)
		if err != nil {
			return err
		}

		var i FleetVehicle
		switch v.Type {
		case "car":
			i = &Car{}
		case "truck":
			i = &Truck{}
		case "bike":
			i = &Bike{}
		default:
			return errors.New("unknown vehicle type")
		}

		err = json.Unmarshal(raw, i)
		if err != nil {
			return err
		}

		f.Vehicles = append(f.Vehicles, i)
	}

	return nil
}

fleet_test.go

import (
	"encoding/json"
	"testing"
)

func TestJSONMarshal(t *testing.T) {

	fleet := &Fleet{
		Vehicles: []FleetVehicle{
			&Car{Vehicle{"car", "Mercedes-Benz", "AMG C 63"}, 4, 250},
			&Bike{Vehicle{"bike", "Triumph", "Bonneville T120"}, 193.12},
		},
	}

	b, err := json.MarshalIndent(&fleet, "", "  ")
	if err != nil {
		t.Fatal(err)
	}

	t.Logf("%s\n", b)
}

func TestJSONUnmarshal(t *testing.T) {

	const body = `{
		"vehicles": [
			{"type": "car", "make": "BMW", "model": "M3", "seatingCapacity": 4, "topSpeed": 250}, 
			{"type": "truck", "make": "Volvo", "model": "FH", "payloadCapacity": 40000},
			{"type": "bike", "make": "Yamaha", "model": "YZF-R1", "topSpeed": 293}
		]
	}`

	var f *Fleet
	err := json.Unmarshal([]byte(body), &f)
	if err != nil {
		t.Fatal(err)
	}

	for _, vehicle := range f.Vehicles {

		if _, ok := vehicle.(FleetVehicle); !ok {
			t.Fatal("expected type to implement FleetVehicle")
		}

		switch v := vehicle.(type) {
		case *Car:
			t.Logf("%s has a seating capacity of %d and a top speed of %.2f km/h\n",
				v.MakeAndModel(),
				v.SeatingCapacity,
				v.TopSpeed)
		case *Truck:
			t.Logf("%s has a payload capacity of %.2f kg\n",
				v.MakeAndModel(),
				v.PayloadCapacity)
		case *Bike:
			t.Logf("%s has a top speed of %.2f km/h\n",
				v.MakeAndModel(),
				v.TopSpeed)
		}
	}
}

fleet_test_output.txt

=== RUN   TestJSONMarshal
--- PASS: TestJSONMarshal (0.00s)
    json_test.go:22: {
          "vehicles": [
            {
              "type": "car",
              "make": "Mercedes-Benz",
              "model": "AMG C 63",
              "seatingCapacity": 4,
              "topSpeed": 250
            },
            {
              "type": "bike",
              "make": "Triumph",
              "model": "Bonneville T120",
              "topSpeed": 193.12
            }
          ]
        }
=== RUN   TestJSONUnmarshal
--- PASS: TestJSONUnmarshal (0.00s)
    json_test.go:45: BMW M3 has a seating capacity of 4 and a top speed of 250.00 km/h
    json_test.go:50: Volvo FH has a payload capacity of 40000.00 kg
    json_test.go:54: Yamaha YZF-R1 has a top speed of 293.00 km/h
PASS

hello.go

package main

import (
	"encoding/json"
	"errors"
	//	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
)

type FleetVehicle interface {
	//	MakeAndModel() string
}

type Vehicle struct {
	Type  string `json:"type"`
	Make  string `json:"make"`
	Model string `json:"model"`
}

// func (v Vehicle) MakeAndModel() string {
// 	return fmt.Sprintf("%s %s", v.Make, v.Model)
// }

type Car struct {
	Vehicle
	SeatingCapacity int     `json:"seatingCapacity"`
	TopSpeed        float64 `json:"topSpeed"`
}

type Truck struct {
	Vehicle
	PayloadCapacity float64 `json:"payloadCapacity"`
}

type Bike struct {
	Vehicle
	TopSpeed float64 `json:"topSpeed"`
}

type Fleet struct {
	Status   string         `json:"status"`
	Vehicles []FleetVehicle `json:"-"`
	//Vehicles []Vehicle `json:"-"`
	//RawVehicles []json.RawMessage
	RawVehicles []json.RawMessage `json:"data"`
}

func (f *Fleet) MarshalJSON() ([]byte, error) {

	type fleet Fleet

	if f.Vehicles != nil {
		for _, v := range f.Vehicles {
			b, err := json.Marshal(v)
			if err != nil {
				return nil, err
			}
			f.RawVehicles = append(f.RawVehicles, b)
		}
	}

	return json.Marshal((*fleet)(f))
}

func (f *Fleet) UnmarshalJSON(b []byte) error {

	type fleet Fleet

	err := json.Unmarshal(b, (*fleet)(f))
	if err != nil {
		return err
	}

	for _, raw := range f.RawVehicles {

		var v Vehicle
		err = json.Unmarshal(raw, &v)
		if err != nil {
			return err
		}

		var i FleetVehicle
		switch v.Type {
		case "car":
			i = &Car{}
		case "truck":
			i = &Truck{}
		case "bike":
			i = &Bike{}
		default:
			return errors.New("unknown vehicle type")
		}

		err = json.Unmarshal(raw, i)
		if err != nil {
			return err
		}

		f.Vehicles = append(f.Vehicles, i)
	}

	return nil
}
func main() {
	http.HandleFunc("/", HelloServer)
	http.ListenAndServe(":8080", nil)
}

// type response2 struct {
// 	Page   int      `json:"page"`
// 	Fruits []string `json:"fruits"`
// }

func Response(w http.ResponseWriter, js []byte) {
	w.Header().Set("Content-Type", "application/json;charset=utf-8")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
	w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization")
	w.Header().Set("Access-Control-Expose-Headers", "Content-Type, X-Requested-With, X-authentication, X-client")
	w.WriteHeader(http.StatusOK)
	w.Write(js)
}

// type ExJsonPath struct {
// 	Status string `json:"status"`
// 	Data   Fleet  `json:"data"`
// }

func LoadFleet(outFile string) Fleet {
	jsonFile, _ := os.Open(outFile)
	defer jsonFile.Close()
	byteValue, _ := ioutil.ReadAll(jsonFile)
	var f *Fleet
	err := json.Unmarshal(byteValue, &f)
	if err != nil {
		log.Fatal(err)
	}
	return *f
}
func HelloServer(w http.ResponseWriter, r *http.Request) {
	// fmt.Fprintf(w, "Hello, %s!", r.URL.Path[1:])
	// es2D := &response2{
	// 	Page:   1,
	// 	Fruits: []string{"apple", "peach", "pear"}}
	// js, _ := json.Marshal(es2D)
	// fleet := &Fleet{
	// 	Vehicles: []FleetVehicle{
	// 		&Car{Vehicle{"car", "Mercedes-Benz", "AMG C 63"}, 4, 250},
	// 		&Bike{Vehicle{"bike", "Triumph", "Bonneville T120"}, 193.12},
	// 	},
	// }
	queryResult := make([]Fleet, 2)
	outFile := "test.json"
	outFile2 := "test2.json"
	queryResult[0] = LoadFleet(outFile)
	queryResult[1] = LoadFleet(outFile2)
	var totalLen int
	for _, s := range queryResult {
		totalLen += len(s.Vehicles)
	}
	tmp := make([]FleetVehicle, totalLen)
	var i int
	for _, s := range queryResult {
		i += copy(tmp[i:], s.Vehicles)
	}
	comboFleet := &Fleet{Vehicles: tmp, Status: "good"}
	js, _ := json.MarshalIndent(&comboFleet, "", "  ")
	//dat := ExJsonPath("success", js)
	Response(w, js)
}

helloVariant.go

package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"varparse"
)

func main() {
	http.HandleFunc("/", HelloServer)
	http.ListenAndServe(":8080", nil)
}

func Response(w http.ResponseWriter, js []byte) {
	w.Header().Set("Content-Type", "application/json;charset=utf-8")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
	w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization")
	w.Header().Set("Access-Control-Expose-Headers", "Content-Type, X-Requested-With, X-authentication, X-client")
	w.WriteHeader(http.StatusOK)
	w.Write(js)
}

func LoadVariants(outFile string) varparse.Variants {
	jsonFile, _ := os.Open(outFile)
	defer jsonFile.Close()
	byteValue, _ := ioutil.ReadAll(jsonFile)
	var f *varparse.Variants
	err := json.Unmarshal(byteValue, &f)
	if err != nil {
		log.Fatal(err)
	}
	return *f
}

func HelloServer(w http.ResponseWriter, r *http.Request) {
	queryResult := make([]varparse.Variants, 2)
	outFile := "testVariant1.json"
	outFile2 := "testVariant2.json"
	queryResult[0] = LoadVariants(outFile)
	queryResult[1] = LoadVariants(outFile2)
	var totalLen int
	for _, s := range queryResult {
		totalLen += len(s.Vars)
	}
	tmp := make([]varparse.VariantInterface, totalLen)
	var i int
	for _, s := range queryResult {
		i += copy(tmp[i:], s.Vars)
	}
	comboVariants := &varparse.Variants{Vars: tmp, Status: "good"}
	js, _ := json.MarshalIndent(&comboVariants, "", "  ")
	Response(w, js)
}

test.json

{"data":[
    {
        "type":"car",
        "make":"Mercedes-Benz",
        "model":"AMG C 63",
        "seatingCapacity":4,
        "topSpeed":250
    },
    {
        "type":"bike",
        "make":"Triumph",
        "model":"Bonneville T120",
        "topSpeed":193.12
    }
]
}

test2.json

{"data":[
    {
        "type":"car",
        "make":"Mercedes-Benz",
        "model":"t2 C 63",
        "seatingCapacity":4,
        "topSpeed":250
    },
    {
        "type":"bike",
        "make":"Triumph",
        "model":"Bonneville t2",
        "topSpeed":193.12
    }
]
}

testVariant1.json

{"data":[
    {
        "chrom":"car",
        "pos":"Mercedes-Benz",
        "ref":"C",
        "type":"VEP_ensembl",
        "vep":"vep_stuff"
    },
    {
        "type":"Annovar_ensembl",
        "chrom":"Triumph",
        "ref":"Triumph",
        "pos":"Bonneville T120",
        "annovar":"annovar_stuff"
    }
]
}

testVariant2.json

{"data":[
    {
        "chrom":"car",
        "pos":"Mercedes-Benz",
        "ref":"C",
        "type":"VEP_ensembl",
        "vep":"vep_stuff2"
    },
    {
        "type":"Annovar_ensembl",
        "chrom":"Triumph",
        "ref":"Triumph",
        "pos":"Bonneville T120",
        "annovar":"annovar_stuff2"
    }
]
}

variant_parsing.go

package varparse

type VariantInterface interface {
}

type Variant struct {
	Chrom string `json:"chrom"`
	Pos   string `json:"pos"`
	Ref   string `json:"ref"`
	Type  string `json:"type"`
}

type VEPEnsembl struct {
	Variant
	Vep string `json:"vep"`
}

type AnnovarEnsembl struct {
	Variant
	Annovar string `json:"annovar"`
}

type Variants struct {
	Status  string             `json:"status"`
	Vars    []VariantInterface `json:"-"`
	RawVars []json.RawMessage  `json:"data"`
}

func (f *Variants) MarshalJSON() ([]byte, error) {

	type variants Variants

	if f.Vars != nil {
		for _, v := range f.Vars {
			b, err := json.Marshal(v)
			if err != nil {
				return nil, err
			}
			f.RawVars = append(f.RawVars, b)
		}
	}

	return json.Marshal((*variants)(f))
}

func (f *Variants) UnmarshalJSON(b []byte) error {

	type variants Variants

	err := json.Unmarshal(b, (*variants)(f))
	if err != nil {
		return err
	}

	for _, raw := range f.RawVariants {

		var v Variant
		err = json.Unmarshal(raw, &v)
		if err != nil {
			return err
		}

		var i VariantInterface
		switch v.Type {
		case "VEPEnsembl":
			i = &VEPEnsembl{}
		case "AnnovarEnsembl":
			i = &AnnovarEnsembl{}
		default:
			return errors.New("unknown tool and database combo")
		}

		err = json.Unmarshal(raw, i)
		if err != nil {
			return err
		}

		f.Variants = append(f.Variants, i)
	}

	return nil
}

varparse.go

package varparse

import (
	"encoding/json"
	"errors"
)

type VariantInterface interface {
}

type Variant struct {
	Chrom string `json:"chrom"`
	Pos   string `json:"pos"`
	Ref   string `json:"ref"`
	Type  string `json:"type"`
}

type VEPEnsembl struct {
	Variant
	Vep string `json:"vep"`
}

type AnnovarEnsembl struct {
	Variant
	Annovar string `json:"annovar"`
}

type Variants struct {
	Status  string             `json:"status"`
	Vars    []VariantInterface `json:"-"`
	RawVars []json.RawMessage  `json:"data"`
}

func (f *Variants) MarshalJSON() ([]byte, error) {

	type variants Variants

	if f.Vars != nil {
		for _, v := range f.Vars {
			b, err := json.Marshal(v)
			if err != nil {
				return nil, err
			}
			f.RawVars = append(f.RawVars, b)
		}
	}

	return json.Marshal((*variants)(f))
}

func (f *Variants) UnmarshalJSON(b []byte) error {

	type variants Variants

	err := json.Unmarshal(b, (*variants)(f))
	if err != nil {
		return err
	}

	for _, raw := range f.RawVars {

		var v Variant
		err = json.Unmarshal(raw, &v)
		if err != nil {
			return err
		}

		var i VariantInterface
		switch v.Type {
		case "VEP_ensembl":
			i = &VEPEnsembl{}
		case "Annovar_ensembl":
			i = &AnnovarEnsembl{}
		default:
			return errors.New("unknown tool and database combo")
		}

		err = json.Unmarshal(raw, i)
		if err != nil {
			return err
		}

		f.Vars = append(f.Vars, i)
	}

	return nil
}

varparse.go

package varparse

type VariantInterface interface {
}

type Variant struct {
	Chrom string `json:"chrom"`
	Pos   string `json:"pos"`
	Ref   string `json:"ref"`
	Type  string `json:"type"`
}

type VEPEnsembl struct {
	Variant
	Vep string `json:"vep"`
}

type AnnovarEnsembl struct {
	Variant
	Annovar string `json:"annovar"`
}

type Variants struct {
	Status  string             `json:"status"`
	Vars    []VariantInterface `json:"-"`
	RawVars []json.RawMessage  `json:"data"`
}

func (f *Variants) MarshalJSON() ([]byte, error) {

	type variants Variants

	if f.Vars != nil {
		for _, v := range f.Vars {
			b, err := json.Marshal(v)
			if err != nil {
				return nil, err
			}
			f.RawVars = append(f.RawVars, b)
		}
	}

	return json.Marshal((*variants)(f))
}

func (f *Variants) UnmarshalJSON(b []byte) error {

	type variants Variants

	err := json.Unmarshal(b, (*variants)(f))
	if err != nil {
		return err
	}

	for _, raw := range f.RawVariants {

		var v Variant
		err = json.Unmarshal(raw, &v)
		if err != nil {
			return err
		}

		var i VariantInterface
		switch v.Type {
		case "VEP_ensembl":
			i = &VEPEnsembl{}
		case "Annovar_ensembl":
			i = &AnnovarEnsembl{}
		default:
			return errors.New("unknown tool and database combo")
		}

		err = json.Unmarshal(raw, i)
		if err != nil {
			return err
		}

		f.Variants = append(f.Variants, i)
	}

	return nil
}