ScrapSaver.go

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"database/sql"
	_ "github.com/lib/pq"
)

type GitHubUser struct {
	ID       int    `json:"id"`
	Username string `json:"login"`
	URL      string `json:"url"`
}

func panicError(err error) {
	if err != nil {
		panic(err)
	}
}

func saveUser(db *sql.DB, user GitHubUser) {
	statement, err := db.Prepare(
		"INSERT INTO public.user (user_id, username, url) VALUES ($1, $2, $3)",
	)
	panicError(err)

	_, err = statement.Exec(user.ID, user.Username, user.URL)
	panicError(err)

	fmt.Println(user)
}

func main() {

	// Connect to database
	db, err := sql.Open("postgres", "user=wdavenport password=efj3f32vnd4oo0vdl dbname=github sslmode=disable")
	panicError(err)

	// Retreive response from GitHub
	response, err := http.Get("https://api.github.com/users?per_page=135")
	panicError(err)

  // Read body of response
	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)
	panicError(err)

	// Convert json to []GitHubUser
	var users []GitHubUser
	err = json.Unmarshal(body, &users)
	panicError(err)

	// Concurrently write to DB
	for _, user := range users {
		go saveUser(db, user)
	}

  // Wait for input to allow goroutines to finish
	var pause string
	fmt.Scan(&pause)

}