Simple thing to download LPA help text occurrences over time

lpa_help.go

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"time"
)

type helpSource struct {
	key   string
	title string
}

var (
	knownHelp = [...]helpSource{
		helpSource{"add-extra-instructions-for-the-attorneys", "attorneys"},
		helpSource{"applying-for-a-reduction-of-the-fee", "reduction"},
		helpSource{"can-i-do-it-all-using-this-tool", "tool"},
		helpSource{"how-long-does-it-take-to-register-an-lpa", "time"},
		helpSource{"how-much-does-it-cost-to-apply-to-register-an-lpa", "cost"},
		helpSource{"i-want-to-pay-by-cheque-or-bacs", "payment"},
		helpSource{"what-does-beta-mean", "beta"},
		helpSource{"what-if-i-want-to-make-more-than-one-lpa", "many"},
	}
)

func main() {
	var helpUriTemplate = "https://www.performance.service.gov.uk/data/lasting-power-of-attorney/journey?filter_by=eventAction%%3Ahelp.inline&group_by=eventDestination&collect=uniqueEvents%%3Asum&period=week&start_at=%s&end_at=%s"
	var volumesUriTemplate = "https://www.performance.service.gov.uk/data/lasting-power-of-attorney/volumes?collect=value%%3Asum&period=week&start_at=%s&end_at=%s&group_by=key"

	printHeader()
	launchDate := time.Date(2013, time.November, 11, 0, 0, 0, 0, time.UTC)
	for date := range generateWeekRange(launchDate) {
		// fmt.Printf("%v\n", date)
		jsonResponse := grabData(helpUriTemplate, date)
		doc := parse(jsonResponse)
		sums := extractSums(doc)
		jsonResponse = grabData(volumesUriTemplate, date)
		doc = parse(jsonResponse)
		volume := extractVolume(doc)
		printResult(date, sums, volume)
	}
}

func generateWeekRange(startDate time.Time) chan time.Time {
	ch := make(chan time.Time)

	endDate := time.Now()

	go func() {
		for i := startDate; i.Before(endDate); i = i.AddDate(0, 0, 7) {
			ch <- i
		}
		close(ch)
	}()

	return ch
}

func grabData(uriTemplate string, startDate time.Time) []byte {
	endDate := startDate.AddDate(0, 0, 7)

	uri := fmt.Sprintf(uriTemplate, urlSafeDate(startDate), urlSafeDate(endDate))

	// fmt.Println(uri)

	res, err := http.Get(uri)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	return body
}

func parse(jsonResponse []byte) map[string]interface{} {
	var doc interface{}
	err := json.Unmarshal(jsonResponse, &doc)
	if err != nil {
		log.Fatal(err)
	}
	return doc.(map[string]interface{})
}

func extractSums(doc map[string]interface{}) map[string]float64 {
	data := doc["data"].([]interface{})
	results := make(map[string]float64)

	for _, record := range data {
		r := record.(map[string]interface{})
		results[r["eventDestination"].(string)] += r["uniqueEvents:sum"].(float64)
	}

	return results
}

func extractVolume(doc map[string]interface{}) float64 {
	res := 0.0
	data := doc["data"].([]interface{})
	for _, record := range data {
		r := record.(map[string]interface{})
		switch r["key"] {
		case "health_and_welfare_digital_applications",
			"property_and_financial_digital_applications":
			{
				res += r["value:sum"].(float64)
			}
		}
	}
	return res
}

func urlSafeDate(date time.Time) string {
	return url.QueryEscape(date.Format("2006-01-02T15:04:05-07:00"))
}

func printHeader() {
	fmt.Printf("               ")
	for _, t := range knownHelp {
		fmt.Printf("%10v", t.title)
	}
	fmt.Println()
}

func printResult(date time.Time, sums map[string]float64, volume float64) {
	fmt.Printf("%v  %3v", date.Format("2006-01-02"), volume)
	for _, t := range knownHelp {
		sum := sums[t.key]
		fmt.Printf("%4v %5.3f", sum, sum/volume)
	}
	fmt.Println()
}