CSV Code Challenge

README.md

CSV Code Challenge

Introduction

To get started run:

go get
go build
./treetop-codechallenge
curl localhost:8080/organizations | python -m json.tool
curl 'localhost:8080/organizations?id=1'
curl 'localhost:8080/organizations?state=CA'
curl 'localhost:8080/organizations?state=CA&orderby=City&direction=ASC'

To run test suite:

go get
go test -v

Requirements

Write an API endpoint that returns a filtered set of organizations from the data provided: https://s3-us-west-2.amazonaws.com/sample-coding-dataset/organization_sample_data.csv Example API:

GET /organizations?category=Greek&city=Washington
Fields:
Id: numeric id
Name: string //organization name
City: string //US city name
State: string //US state name
Postal: string //US postal code
Category: string //categorization of org
Additional query params:
Orderby: string //fieldname to order the results by
Direction: string //ASC or DSC

The expected response is a JSON collection of organizations:

{
	"organizations": [
		{
			"id":"102",
			"name":"Sigma Kappa Zeta Chapter",
			"city":"Washington",
			"state":"DC",
			"postal":"20052",
			"category":"Greek"
		},
		...
	]
}

All query parameters are optional.

Design

• After downloading the csv, the size is negligable enough to read at startup and then keep in memory for query
• I plan to use the encoding/csv package to read in the data
• I plan to use the sort package to sort the objects
• I'll keep an initial slice with all the objects serialized from the csv
• I'll create a new slice and reference the original object in the filtered and sorted slice
• If no direction is specified, or something other than ASC or DSC then ASC will be the default

Testing

I decided to break down the project into these testable components:

1. Parsing the CSV
2. Filtering
3. Sorting

I wrote some simple tests that ensure that I properly parse the downloaded csv document. I also wrote tests around filtering and sorting.

Implementation

• On bootup I'll download and parse the csv file into a slice of Organization References
• I'll then start serving on port 8080, and have a Handler that filters and sorts a new array.
• Ran into difficulties with the csv, it was using only carriage return characters, with no new line chars, thus not conforming to the standard
• Decided to just download and modify the dataset locally, used the vim command: :%s/\r/\r/g

Improvements

Ideally this information would be stored in some sort of database that could do the filtering and sorting through SQL. I could improve performance by storing references in datastructures like trees or maps to optimize lookups for certain attributes. I also could write some better tests that would cover my code more comprehensively. It also looks like some null values are getting through, I could account for that in the csv import

filter.go

package main

import (
	"net/url"
	"strconv"
)

func (oh *OrganizationsHandler) Filter(v url.Values) []*Organization {
	os := make([]*Organization, 0)
	for _, o := range oh.Organizations {
		match := true
		if v.Get("id") != "" &&
			v.Get("id") != strconv.Itoa(o.ID) {
			match = false
		}
		if v.Get("name") != "" &&
			v.Get("name") != o.Name {
			match = false
		}
		if v.Get("city") != "" &&
			v.Get("city") != o.City {
			match = false
		}
		if v.Get("state") != "" &&
			v.Get("state") != o.State {
			match = false
		}
		if v.Get("postal") != "" &&
			v.Get("postal") != o.Postal {
			match = false
		}
		if v.Get("category") != "" &&
			v.Get("category") != o.Category {
			match = false
		}
		if match {
			os = append(os, o)
		}
	}
	return os
}

filter_test.go

package main

import (
	"net/url"
	"testing"
)

func TestFilterIDs(t *testing.T) {
	values := url.Values{}
	id1 := new(Organization)
	id1.ID = 1
	id2 := new(Organization)
	id2.ID = 2
	ids := []*Organization{id1, id2}

	oh := NewOrganizationsHandler(ids)
	values.Set("id", "1")
	ids = oh.Filter(values)

	if len(ids) != 1 {
		t.Error("Should be only one filtered org")
	}
}

handler.go

package main

import (
	"encoding/json"
	"net/http"
)

type OrganizationsHandler struct {
	Organizations []*Organization
}

func NewOrganizationsHandler(os []*Organization) *OrganizationsHandler {
	oh := new(OrganizationsHandler)
	oh.Organizations = os
	return oh
}

func (oh *OrganizationsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	//Filter the objects
	os := oh.Filter(r.URL.Query())

	//Sort the objects
	os = Sort(r.URL.Query(), os)

	e := json.NewEncoder(w)
	e.Encode(&os)
}

main.go

package main

import (
	"fmt"
	"net/http"
	"os"
)

type Organization struct {
	//Id numeric id
	ID int `json:"id"`
	//Name organization name
	Name string `json:"name"`
	//City US city name
	City string `json:"city"`
	//US state name
	State string `json:"state"`
	//US postal code
	Postal string `json:"postal"`
	//categorization of org
	Category string `json:"category"`
}

func main() {
	//On startup download and parse the csv file
	f, err := os.Open("sampledata.csv")
	if err != nil {
		fmt.Println(err)
		return
	}
	ozs, err := parse(f)
	if err != nil {
		fmt.Println(err)
		return
	}

	//Create a new handler
	oh := NewOrganizationsHandler(ozs)

	//Attach it to http
	http.Handle("/organizations", oh)

	http.ListenAndServe(":8080", nil)
}

parser.go

package main

import (
	"encoding/csv"
	"io"
	"strconv"
)

// parse will take an io reader, run through it and build an array of objects
func parse(r io.Reader) ([]*Organization, error) {
	os := make([]*Organization, 0)
	cr := csv.NewReader(r)
	for {
		record, err := cr.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return os, err
		}
		o := new(Organization)
		o.ID, _ = strconv.Atoi(record[0])
		o.Name = record[1]
		o.City = record[2]
		o.State = record[3]
		o.Postal = record[4]
		o.Category = record[5]
		if o.ID > 0 {
			os = append(os, o)
		}
	}
	return os, nil
}

parser_test.go

package main

import (
	"os"
	"testing"
)

func TestParse(t *testing.T) {
	f, err := os.Open("sampledata.csv")
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	organizations, err := parse(f)
	if err != nil {
		t.Fatal(err)
	}

	if len(organizations) != 200 {
		t.Error("Organizations must contain 200 objects")
	}

	for _, o := range organizations {
		if o.ID <= 0 {
			t.Error("Organization should have a positive id")
		}
		if o.Name == "" {
			t.Error("Organization should have a name")
		}
	}
}

sampledata.csv

sort.go

package main

import (
	"net/url"
	"sort"
)

func Sort(v url.Values, os []*Organization) []*Organization {
	by := v.Get("orderby")
	dir := v.Get("direction")
	if dir == "" || dir != "DSC" {
		dir = "ASC"
	}
	switch by {
	case "id":
		if dir == "ASC" {
			sort.Slice(os, func(i, j int) bool { return os[i].ID < os[j].ID })
		} else {
			sort.Slice(os, func(i, j int) bool { return os[i].ID > os[j].ID })
		}
	case "name":
		if dir == "ASC" {
			sort.Slice(os, func(i, j int) bool { return os[i].Name < os[j].Name })
		} else {
			sort.Slice(os, func(i, j int) bool { return os[i].Name > os[j].Name })
		}
	case "city":
		if dir == "ASC" {
			sort.Slice(os, func(i, j int) bool { return os[i].City < os[j].City })
		} else {
			sort.Slice(os, func(i, j int) bool { return os[i].City > os[j].City })
		}
	case "state":
		if dir == "ASC" {
			sort.Slice(os, func(i, j int) bool { return os[i].State < os[j].State })
		} else {
			sort.Slice(os, func(i, j int) bool { return os[i].State > os[j].State })
		}
	case "postal":
		if dir == "ASC" {
			sort.Slice(os, func(i, j int) bool { return os[i].Postal < os[j].Postal })
		} else {
			sort.Slice(os, func(i, j int) bool { return os[i].Postal > os[j].Postal })
		}
	case "category":
		if dir == "ASC" {
			sort.Slice(os, func(i, j int) bool { return os[i].Category < os[j].Category })
		} else {
			sort.Slice(os, func(i, j int) bool { return os[i].Category > os[j].Category })
		}
	}

	return os
}

sort_test.go

package main

import (
	"net/url"
	"testing"
)

func TestSortIDsDSC(t *testing.T) {
	//Test that id is sorted correctly
	values := url.Values{}

	id1 := new(Organization)
	id1.ID = 1
	id2 := new(Organization)
	id2.ID = 2
	ids := []*Organization{id1, id2}

	values.Set("orderby", "id")
	values.Set("direction", "DSC")
	ids = Sort(values, ids)

	if ids[0].ID < ids[1].ID {
		t.Error("Ids not sorted in DSC correctly")
	}
}