[Language Processing 100 Essentials] #60: Parse JSON file and set name and area of artists on key-value store

set_keyval.go

package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"

	"github.com/go-redis/redis"
)

type Artist struct {
	ID       int       `json:"id"`
	GID      string    `json:"gid"`
	Name     string    `json:"name"`
	SortName string    `json:"sort_name"`
	Area     string    `json:"area"`
	Aliases  []*Aliase `json:"aliases"`
	Begin    *Begin    `json:"begin"`
	End      *End      `json:"end"`
	Tags     []*Tag    `json:"tags"`
	Rating   *Rating   `json:"rating"`
}

type Aliase struct {
	Name     string `json:"name"`
	SortName string `json:"sort_name"`
}

type Begin struct {
	Year  int `json:"year"`
	Month int `json:"month"`
	Date  int `json:"date"`
}

type End struct {
	Year  int `json:"year"`
	Month int `json:"month"`
	Date  int `json:"date"`
}

type Tag struct {
	Count int    `json:"count"`
	Value string `json:"value"`
}

type Rating struct {
	Count int `json:"count"`
	Value int `json:"value"`
}

func main() {
	var filepath string
	flag.StringVar(&filepath, "file", "", "specify a file path")
	flag.StringVar(&filepath, "f", "", "specify a file path")
	flag.Parse()

	artists, err := readJSON(filepath)
	if err != nil {
		fmt.Print(err)
		os.Exit(1)
	}

	client, err := newRedisClient()
	if err != nil {
		fmt.Print(err)
		os.Exit(1)
	}

	for _, artist := range artists {
		val, err := client.Get(artist.Name).Result()
		if err != nil {
			fmt.Print(err)
			os.Exit(1)
		}
		if err != redis.Nil {
			fmt.Printf("(key, val) = (%s, %s)\n", artist.Name, val)
			continue
		}

		err = client.Set(artist.Name, artist.Area, 0).Err()
		if err != nil {
			fmt.Print(err)
			os.Exit(1)
		}
	}
}

func readJSON(path string) ([]*Artist, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("could not open a file: %s\n  %s", path, err)
	}
	defer f.Close()

	var artists []*Artist
	reader := bufio.NewReader(f)
	for {
		artist := Artist{}
		buf, readErr := reader.ReadBytes('\n')
		if (readErr != nil) && (readErr != io.EOF) {
			panic(err)
		}

		if err = json.Unmarshal(buf, &artist); err != nil && readErr != io.EOF {
			fmt.Print("could not parse json file.")
			break
		}

		artists = append(artists, &artist)

		if readErr == io.EOF {
			break
		}
	}
	return artists, nil
}

func newRedisClient() (*redis.Client, error) {
	client := redis.NewClient(&redis.Options{
		Addr:     "127.0.0.1:6379",
		Password: "",
		DB:       0,
	})

	pong, err := client.Ping().Result()
	if err != nil {
		return nil, fmt.Errorf("could not connect to redis\n  %s", err)
	}

	fmt.Println(pong, err)

	return client, nil
}

set_keyval_test.go

package main

import (
	"os"
	"testing"

	"github.com/go-redis/redis"
	"github.com/go-test/deep"
)

var readJSONTests = []struct {
	name string
	file string
	text string
	want []*Artist
}{
	{
		name: "should parse JSON from full text",
		file: "./fulltext-test.json",
		text: `{"name": "WIK▲N", "tags": [{"count": 1, "value": "sillyname"}], "sort_name": "WIK▲N", "ended": true, "gid": "8972b1c1-6482-4750-b51f-596d2edea8b1", "id": 805192}
{"name": "Gustav Ruppke", "sort_name": "Gustav Ruppke", "ended": true, "gid": "b4f76788-7e6f-41b7-ac7b-dfb67f66282e", "type": "Person", "id": 578352}`,
		want: []*Artist{
			&Artist{
				ID:   805192,
				GID:  "8972b1c1-6482-4750-b51f-596d2edea8b1",
				Name: "WIK▲N",
				Tags: []*Tag{
					&Tag{
						Count: 1,
						Value: "sillyname",
					},
				},
				SortName: "WIK▲N",
			},
			&Artist{
				ID:       578352,
				GID:      "b4f76788-7e6f-41b7-ac7b-dfb67f66282e",
				Name:     "Gustav Ruppke",
				SortName: "Gustav Ruppke",
			},
		},
	},
	{
		name: "should not parse JSON from empty text",
		file: "./empty-test.json",
		text: ``,
		want: []*Artist{
			&Artist{},
		},
	},
}

func TestReadJSON(t *testing.T) {
	for _, testcase := range readJSONTests {
		t.Log(testcase.name)

		f, err := os.Create(testcase.file)
		if err != nil {
			t.Errorf("could not create a file: %s\n  %s\n", testcase.file, err)
		}
		f.WriteString(testcase.text)
		f.Close()

		results, err := readJSON(testcase.file)
		if err != nil {
			t.Errorf("could not parse a JSON file: %s\n  %s\n", testcase.file, err)
		}

		if diff := deep.Equal(results, testcase.want); diff != nil {
			t.Error(diff)
		}

		if err := os.Remove(testcase.file); err != nil {
			t.Errorf("could not delete a file: %s\n  %s\n", testcase.file, err)
		}
	}
}

var getKeyValTests = []struct {
	name string
	file string
	text string
	want []string
}{
	{
		name: "should parse JSON from full text",
		file: "./fulltext-test.json",
		text: `{"begin": {"year": 1956}, "end": {"year": 1993}, "name": "The Silhouettes", "area": "United States", "sort_name": "Silhouettes, The", "ended": true, "gid": "ca3f3ee1-c4a7-4bac-a16a-0b888a396c6b", "type": "Group", "id": 101060, "aliases": [{"name": "Silhouettes", "sort_name": "Silhouettes"}, {"name": "The Sihouettes", "sort_name": "The Sihouettes"}]}
{"name": "Gustav Ruppke", "sort_name": "Gustav Ruppke", "ended": true, "gid": "b4f76788-7e6f-41b7-ac7b-dfb67f66282e", "type": "Person", "id": 578352}`,
		want: []string{
			"United States",
			"",
		},
	},
	{
		name: "should not parse JSON from empty text",
		file: "./empty-test.json",
		text: ``,
		want: []string{""},
	},
}

func TestGetKeyVal(t *testing.T) {
	for _, testcase := range getKeyValTests {
		t.Log(testcase.name)

		f, err := os.Create(testcase.file)
		if err != nil {
			t.Errorf("could not create a file: %s\n  %s\n", testcase.file, err)
		}
		f.WriteString(testcase.text)
		f.Close()

		artists, err := readJSON(testcase.file)
		if err != nil {
			t.Errorf("could not parse a JSON file: %s\n  %s\n", testcase.file, err)
		}

		client, err := newRedisClient()
		if err != nil {
			t.Error(err)
		}

		var results []string
		for _, artist := range artists {
			err = client.Set(artist.Name, artist.Area, 0).Err()
			if err != nil {
				t.Error(err)
			}

			val, err := client.Get(artist.Name).Result()
			if err != nil {
				t.Error(err)
			}
			if err != redis.Nil {
				results = append(results, val)
			}
		}

		if diff := deep.Equal(results, testcase.want); diff != nil {
			t.Error(diff)
		}

		if err := os.Remove(testcase.file); err != nil {
			t.Errorf("could not delete a file: %s\n  %s\n", testcase.file, err)
		}
	}
}