Read the go lock files from all the repos of a given org, find the deps licenses from github api and dump the result in a csv file

licenses.go

package main

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strings"
)

const (
	ghDomainName = "github.com"
	oauthToken   = ""
	lockfileName = "Gopkg.lock"
	githubURL    = "https://api.github.com/repos/%s/%s/license?access_token=%s"
	resultFile   = "licenses.csv"
)

var (
	reGithubUrl  = regexp.MustCompile("name = \"(.*?)\"")
	reGithubInfo = regexp.MustCompile("github.com/(.*?)/(.*?)")
)

type dependency struct {
	name    string
	owner   string
	url     string
	license string
}

func (d dependency) string() string {
	return fmt.Sprintf("%s/%s", d.owner, d.name)
}

func newDependency(name, owner, url string) dependency {
	return dependency{
		name:  name,
		owner: owner,
		url:   url,
	}
}

type githubAPIResponse struct {
	License struct {
		Key string `json:key`
	} `json:license`
}

func main() {
	if len(os.Args) < 2 {
		log.Fatal("you need to specify a valid go organization path")
	}

	orgDeps := map[string]dependency{}
	orgPath := strings.TrimSuffix(os.Args[1], "/")
	log.Printf("retrieving licenses for organization: %s", orgPath)

	projectPaths := findProjectPaths(orgPath)
	for _, path := range projectPaths {
		deps := findProjectDependencies(path)
		for _, dep := range deps {
			if _, ok := orgDeps[dep.string()]; !ok {
				orgDeps[dep.string()] = dep
			}
		}
	}

	log.Printf("dumping to result file...")
	dumpDependencies(orgDeps)

	log.Printf("done")
}

func findProjectPaths(orgPath string) (projectPaths []string) {
	files, err := ioutil.ReadDir(orgPath)
	if err != nil {
		log.Fatalf("failed reading directory: %s", err)
	}

	for _, f := range files {
		if f.IsDir() {
			path := fmt.Sprintf("%s/%s", orgPath, f.Name())
			projectPaths = append(projectPaths, path)
		}
	}
	return
}

func findProjectDependencies(path string) []dependency {
	log.Printf("retrieving licenses for repository: %s", path)

	lockfilePath := fmt.Sprintf("%s/%s", path, lockfileName)
	lockLines := readLines(lockfilePath)
	nameLines := filterNameLines(lockLines)
	log.Printf("found %d dependencies", len(nameLines))

	ghDeps := extractGithubDependencies(nameLines)
	log.Printf("found %d github dependencies", len(ghDeps))

	licensedDeps := addGithubLicenses(ghDeps)
	log.Printf("found %d licenses", len(licensedDeps))

	return licensedDeps
}

func readLines(path string) (lines []string) {
	file, err := os.Open(path)
	if err != nil {
		log.Fatalf("failed opening file: %s", err)
	}

	defer file.Close()

	scanner := bufio.NewScanner(file)
	scanner.Split(bufio.ScanLines)

	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	return
}

func filterNameLines(lines []string) (nameLines []string) {
	for i := range lines {
		line := strings.TrimSpace(lines[i])
		if strings.HasPrefix(line, "name") {
			nameLines = append(nameLines, line)
		}
	}
	return
}

func extractGithubDependencies(lines []string) (deps []dependency) {
	for i := range lines {
		urlMatch := reGithubUrl.FindStringSubmatch(lines[i])
		if len(urlMatch) < 2 {
			continue
		}

		url := urlMatch[1]
		infos := strings.Split(url, "/")
		if len(infos) < 3 || infos[0] != ghDomainName {
			continue
		}

		url = fmt.Sprintf("https://%s", url)
		dep := newDependency(infos[2], infos[1], url)
		deps = append(deps, dep)
	}
	return
}

func addGithubLicenses(deps []dependency) (licensedDeps []dependency) {
	for _, dep := range deps {
		license, err := queryGithubLicenceAPI(dep.owner, dep.name)
		if err != nil {
			log.Printf("error querying github api: %s", err.Error())
			continue
		}

		dep.license = license
		licensedDeps = append(licensedDeps, dep)
	}
	return
}

func queryGithubLicenceAPI(owner, repo string) (string, error) {
	var githubAPIResponse githubAPIResponse

	url := fmt.Sprintf(githubURL, owner, repo, oauthToken)
	resp, err := http.Get(url)
	if err != nil {
		return githubAPIResponse.License.Key, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		msg := fmt.Sprintf("status %d - owner %s - repo %s", resp.StatusCode, owner, repo)
		return githubAPIResponse.License.Key, errors.New(msg)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return githubAPIResponse.License.Key, err
	}
	err = json.Unmarshal(body, &githubAPIResponse)
	return githubAPIResponse.License.Key, err
}

func dumpDependencies(deps map[string]dependency) {
	file, err := os.Create(resultFile)
	if err != nil {
		log.Fatalf("failed creating result file: %s", err)
	}
	defer file.Close()

	writer := csv.NewWriter(file)
	defer writer.Flush()

	for _, dep := range deps {
		line := []string{dep.string(), dep.license, dep.url}
		if err := writer.Write(line); err != nil {
			log.Printf("error writing to result file: %s", err.Error())
			continue
		}
	}
}