Lagoon Storage Charts

README.md

PoC Lagoon Environment Storage Charts

Generates a set of line charts to plot detailed storage information for all environments in a project. Can help when comparing the compiled data in a billing tool to the raw data from Lagoon.

Instructions

1. Get a copy of the storage data from the Lagoon API and save it as a file.

   query($name: String!) {
     projectByName(name: $name) {
       name
       environments(includeDeleted: true) {
         name
         deleted
         storages {
           persistentStorageClaim
           bytesUsed
           updated
         }
       }
     }
   }

2. Generate a basic report.

   go run . <filename>
   

3. Open the generated HTML file in a browser.

CLI Arguments

Usage: go run . <filename> <included storage in GB>

If you pass the amount of storage the project gets for free, a new line will be added to the totals chart to show the amount of overage for the project.

go.mod

module lagoon/storage-data

go 1.21.4

require (
	github.com/go-echarts/go-echarts/v2 v2.3.2 // indirect
	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
)

go.sum

github.com/go-echarts/go-echarts/v2 v2.3.2 h1:imRxqF5sLtEPBsv5HGwz9KklNuwCo0fTITZ31mrgfzo=
github.com/go-echarts/go-echarts/v2 v2.3.2/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI=
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=

main.go

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"math"
	"os"
	"slices"
	"sort"
	"strconv"
	"time"

	"github.com/go-echarts/go-echarts/v2/charts"
	"github.com/go-echarts/go-echarts/v2/components"
	"github.com/go-echarts/go-echarts/v2/opts"
	"golang.org/x/exp/maps"
)

// Types to represent the return data of the Lagoon API.
//
// {
// 	"data": {
// 		"projectByName": {
// 			"id": 1648,
// 			"name": "example-com",
// 			"environments": [
// 				{
// 					"id": 231314,
// 					"name": "dev",
// 					"deleted": "0000-00-00 00:00:00",
// 					"environmentType": "development",
// 					"storages": [
// 						{
// 							"id": 3913612,
// 							"persistentStorageClaim": "mariadb",
// 							"bytesUsed": 5325968,
// 							"updated": "2019-12-13"
// 						},

type Storage struct {
	Type      string `json:"persistentStorageClaim"`
	KiloBytes int    `json:"bytesUsed"`
	Updated   string `json:"updated"`
}

type Environment struct {
	Name     string    `json:"name"`
	Deleted  string    `json:"deleted"`
	Storages []Storage `json:"storages"`
}

type Project struct {
	Name         string        `json:"name"`
	Environments []Environment `json:"environments"`
}

type RespData struct {
	ProjectByName Project `json:"projectByName"`
}

type RespErrors struct {
}

type ApiResponse struct {
	Data RespData `json:"data"`
}

// Type to represent the converted raw API data.
type CalculatedData map[string]map[string]int

func kbToGb(kb int) float64 {
	return math.Round(float64(kb)/10000) / 100
}

func gbToKb(gb float64) int {
	return int(gb * 1000000)
}

// Chart a line for a given storage Type.
func generateTypeLine(data CalculatedData, dates []string, Type string) []opts.LineData {
	items := make([]opts.LineData, 0)
	for _, date := range dates {
		kb, exists := data[date][Type]
		if exists {
			items = append(items, opts.LineData{Value: []interface{}{date, kbToGb(kb)}})
		} else {
			items = append(items, opts.LineData{Value: []interface{}{date, 0}})
		}
	}

	return items
}

// Chart a line that sums all storage Types.
func generateTotalLine(data CalculatedData, dates []string) []opts.LineData {
	items := make([]opts.LineData, 0)
	for _, date := range dates {
		types := maps.Values(data[date])
		typesTotal := 0
		for _, kb := range types {
			typesTotal += kb
		}

		items = append(items, opts.LineData{Value: []interface{}{date, kbToGb(typesTotal)}})
	}

	return items
}

// Chart a line with a rolling average of the total.
func generateAverageLine(data CalculatedData, dates []string, days int) []opts.LineData {
	dayBuffer := make([]int, days)

	items := make([]opts.LineData, 0)
	for _, date := range dates {
		types := maps.Values(data[date])
		typesTotal := 0
		for _, kb := range types {
			typesTotal += kb
		}

		dayBuffer = append(dayBuffer[1:], typesTotal)

		daysTotal := 0
		for _, kb := range dayBuffer {
			daysTotal += kb
		}

		items = append(items, opts.LineData{Value: []interface{}{date, kbToGb(daysTotal / days)}})
	}

	return items
}

// Chart a line for a rolling sum of the total.
// When passed `allotted`, subtract that amount to show the total "overage."
func generateRollingTotalLine(data CalculatedData, dates []string, days int, allotted float64) []opts.LineData {
	dayBuffer := make([]int, days)

	items := make([]opts.LineData, 0)
	for _, date := range dates {
		types := maps.Values(data[date])
		typesTotal := 0
		for _, kb := range types {
			typesTotal += max(kb-gbToKb(allotted), 0)
		}

		dayBuffer = append(dayBuffer[1:], typesTotal)

		daysTotal := 0
		for _, kb := range dayBuffer {
			daysTotal += kb
		}

		items = append(items, opts.LineData{Value: []interface{}{date, kbToGb(daysTotal)}})
	}

	return items
}

// Chart a line with a billable month total.
func generateMonthAverageLine(data CalculatedData, dates []string) []opts.LineData {
	monthTotal := 0
	items := make([]opts.LineData, 0)
	for i, date := range dates {
		types := maps.Values(data[date])
		typesTotal := 0
		for _, kb := range types {
			typesTotal += kb
		}

		monthTotal += typesTotal

		today, _ := time.Parse(time.DateOnly, dates[i])
		var tomorrow time.Time
		if len(dates) == i+1 {
			tomorrow = today.AddDate(0, 1, 0)
		} else {
			tomorrow, _ = time.Parse(time.DateOnly, dates[i+1])
		}

		if today.Month() != tomorrow.Month() {
			t := time.Date(today.Year(), today.Month(), 32, 0, 0, 0, 0, time.UTC)
			daysInMonth := 32 - t.Day()
			// If it's the last day of the month (that we have data), add a datapoint
			items = append(items, opts.LineData{Value: []interface{}{date, kbToGb(monthTotal / daysInMonth)}})
			// and reset the counter
			monthTotal = 0
		} else {
			items = append(items, opts.LineData{Value: []interface{}{date, nil}})
		}

	}

	return items
}

// Chart a line with a month excess over allotted.
func generateMonthTotalLine(data CalculatedData, dates []string, allotted float64) []opts.LineData {
	monthTotal := 0
	items := make([]opts.LineData, 0)
	for i, date := range dates {
		types := maps.Values(data[date])
		typesTotal := 0
		for _, kb := range types {
			typesTotal += max(kb-gbToKb(allotted), 0)
		}

		monthTotal += typesTotal

		today, _ := time.Parse(time.DateOnly, dates[i])
		var tomorrow time.Time
		if len(dates) == i+1 {
			tomorrow = today.AddDate(0, 1, 0)
		} else {
			tomorrow, _ = time.Parse(time.DateOnly, dates[i+1])
		}

		if today.Month() != tomorrow.Month() {
			// If it's the last day of the month (that we have data), add a datapoint
			items = append(items, opts.LineData{Value: []interface{}{date, kbToGb(monthTotal)}})
			// and reset the counter
			monthTotal = 0
		} else {
			items = append(items, opts.LineData{Value: []interface{}{date, nil}})
		}

	}

	return items
}

// Common line chart base.
func createLineChart(project string, title opts.Title) *charts.Line {
	line := charts.NewLine()

	line.SetGlobalOptions(
		charts.WithTitleOpts(title),
		charts.WithInitializationOpts(opts.Initialization{
			PageTitle: fmt.Sprintf("%s | Lagoon Storage", project),
			Width:     "2400px",
			Height:    "500px",
		}),
		charts.WithYAxisOpts(opts.YAxis{
			Name:         "Use (GB)",
			NameLocation: "center",
			Type:         "value",
		}),
		charts.WithXAxisOpts(opts.XAxis{
			Type: "time",
		}),
		charts.WithDataZoomOpts(opts.DataZoom{
			Type: "slider",
		}),
		charts.WithTooltipOpts(opts.Tooltip{Show: true, Trigger: "axis"}),
	)

	return line
}

// Generate a chart summing all environments.
func allEnvsChart(resp ApiResponse, dates []string) *charts.Line {
	types := []string{}
	calcData := CalculatedData{}
	for _, env := range resp.Data.ProjectByName.Environments {
		for _, stor := range env.Storages {
			if !slices.Contains(types, stor.Type) {
				types = append(types, stor.Type)
			}

			_, dateExists := calcData[stor.Updated]
			if !dateExists {
				calcData[stor.Updated] = map[string]int{}
			}

			calcData[stor.Updated][stor.Type] += stor.KiloBytes
		}
	}

	line := createLineChart(resp.Data.ProjectByName.Name, opts.Title{
		Title:    fmt.Sprintf("%s\n%s — %s", resp.Data.ProjectByName.Name, dates[0], dates[len(dates)-1]),
		Subtitle: "All Environments",
	})

	for _, Type := range types {
		line.AddSeries(Type, generateTypeLine(calcData, dates, Type),
			charts.WithLineStyleOpts(opts.LineStyle{
				Type: "dashed",
			}))
	}

	line.
		AddSeries("Daily Total", generateTotalLine(calcData, dates)).
		AddSeries("30 Day Avg", generateAverageLine(calcData, dates, 30)).
		AddSeries("Month End Avg", generateMonthAverageLine(calcData, dates),
			charts.WithLineChartOpts(opts.LineChart{
				ConnectNulls: true,
				ShowSymbol:   true,
			}),
			charts.WithLineStyleOpts(opts.LineStyle{
				Width: 5,
			}))

	if len(os.Args) > 2 {
		includedFreeStorage, _ := strconv.ParseFloat(os.Args[2], 64)
		line.
			AddSeries("Month End Excess", generateMonthTotalLine(calcData, dates, includedFreeStorage),
				charts.WithLineChartOpts(opts.LineChart{
					ConnectNulls: true,
					ShowSymbol:   true,
				}))

	}

	return line
}

// Generate a chart for one environment.
func envChart(resp ApiResponse, dates []string, env Environment) *charts.Line {
	types := []string{}
	calcData := CalculatedData{}
	for _, stor := range env.Storages {
		if !slices.Contains(types, stor.Type) {
			types = append(types, stor.Type)
		}

		_, dateExists := calcData[stor.Updated]
		if !dateExists {
			calcData[stor.Updated] = map[string]int{}
		}

		calcData[stor.Updated][stor.Type] += stor.KiloBytes
	}

	line := createLineChart(resp.Data.ProjectByName.Name, opts.Title{
		Title: env.Name,
	})

	for _, Type := range types {
		line.AddSeries(Type, generateTypeLine(calcData, dates, Type),
			charts.WithLineStyleOpts(opts.LineStyle{
				Type: "dashed",
			}))
	}

	line.AddSeries("Total", generateTotalLine(calcData, dates))

	return line
}

func main() {

	if len(os.Args) < 2 {
		fmt.Println("Usage: go run . <filename> <included storage in GB>")
		return
	}

	loadFile := os.Args[1]
	fmt.Println("Generating charts from " + loadFile)

	content, err := os.ReadFile(loadFile)
	if err != nil {
		log.Fatal("Error when opening file: ", err)
	}

	var resp ApiResponse
	err = json.Unmarshal(content, &resp)
	if err != nil {
		fmt.Printf("Could not unmarshal json: %s\n", err)
		return
	}

	// Generate a list of all dates with recorded storage.
	datesMap := map[string]bool{}
	for _, env := range resp.Data.ProjectByName.Environments {
		for _, stor := range env.Storages {
			_, dateExists := datesMap[stor.Updated]
			if !dateExists {
				datesMap[stor.Updated] = true
			}
		}
	}

	dates := maps.Keys(datesMap)
	sort.Strings(dates)

	page := components.NewPage()
	page.SetLayout(components.Layout(components.PageCenterLayout))

	page.AddCharts(allEnvsChart(resp, dates))

	for _, env := range resp.Data.ProjectByName.Environments {
		page.AddCharts(envChart(resp, dates, env))
	}

	saveFile := fmt.Sprintf("storage-%s.html", resp.Data.ProjectByName.Name)
	f, _ := os.Create(saveFile)
	_ = page.Render(f)

	fmt.Println("Charts saved as " + saveFile)
}