Gopkg.lock & go.mod compare

go.mod

module github.com/Helcaraxan/dep-list

require (
	github.com/BurntSushi/toml v0.3.1
	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
	github.com/sirupsen/logrus v1.4.0
	github.com/stretchr/testify v1.3.0 // indirect
	golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c // indirect
	golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc // indirect
)

main.go

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"regexp"
	"strings"

	"github.com/BurntSushi/toml"
	"github.com/sirupsen/logrus"
)

var (
	modDepRE  = regexp.MustCompile(`^[^@\s]+(?:@[^@\s]+)? ([^@\s]+)@[^@\s]+$`)
	modListRE = regexp.MustCompile(`^([^\s]+)(?: ([^\s]+))?(?: => ([^\s]+) ([^@\s]+))?$`)
	versionRE = regexp.MustCompile(`^(v[^-\+]+)(?:\+incompatible)?(?:-[^-]+-([^-]{12}))?$`)

	logger = logrus.New()
)

type DepLock struct {
	Projects []DepProject
}

type DepProject struct {
	Name     string
	Source   string
	Revision string
}

type Dep struct {
	replacement string
	version     string
}

// This binary must be run within a Go module and takes 3 mandatory arguments:
// - Path to the module's pre-existing Gopkg.lock file.
// - Private access-token for your GH account.
// - Private access-token for your GitLab account.
func main() {
	lockInfo := &DepLock{}
	if _, err := toml.DecodeFile(os.Args[1], lockInfo); err != nil {
		panic(err)
	}

	lockDeps := map[string]*Dep{}
	for _, project := range lockInfo.Projects {
		newDep := &Dep{version: project.Revision[:12]}

		source := strings.TrimSuffix(strings.TrimPrefix(project.Source, "https://"), ".git")
		if len(source) != 0 && source != project.Name {
			newDep.replacement = source
		}
		lockDeps[project.Name] = newDep
	}

	modDeps := map[string]*Dep{}
	if err := getDirectDeps(modDeps); err != nil {
		panic(err)
	}

	if err := getDirectDepsVersions(modDeps); err != nil {
		panic(err)
	}

	if err := resolveVersions(modDeps); err != nil {
		panic(err)
	}

	for name, dep := range modDeps {
		lock, ok := lockDeps[name]
		if !ok {
			logger.Warnf("Missing dependency in lock: %q - %+v\n", name, dep)
			continue
		}

		if lock.replacement != dep.replacement {
			logger.Warnf("Dependency %q has differing replacements: %q - %q\n", name, dep.replacement, lock.replacement)
		}
		if lock.version != dep.version {
			logger.Warnf("Dependency %q has differing versions: %q - %q\n", name, dep.version, lock.version)
		}
	}
}

func getDirectDeps(deps map[string]*Dep) error {
	logger.Infof("Fetching direct deps")

	cmd := exec.Command("go", "mod", "graph")
	raw, err := cmd.CombinedOutput()
	if err != nil {
		return err
	}

	for _, dep := range strings.Split(string(raw), "\n") {
		if !strings.HasPrefix(dep, "improbable.io/platform") {
			continue
		}

		depContent := modDepRE.FindStringSubmatch(dep)
		if len(depContent) == 0 {
			continue
		}
		deps[depContent[1]] = &Dep{}
	}
	return nil
}

func getDirectDepsVersions(deps map[string]*Dep) error {
	logger.Infof("Fetching direct dep versions")

	cmd := exec.Command("go", "list", "-m", "all")
	raw, err := cmd.CombinedOutput()
	if err != nil {
		return err
	}

	for _, dep := range strings.Split(string(raw), "\n") {
		if len(dep) == 0 {
			continue
		}

		depContent := modListRE.FindStringSubmatch(strings.TrimSpace(dep))
		if len(depContent) == 0 {
			return fmt.Errorf("unusual go list line: %q", dep)
		}

		details, ok := deps[depContent[1]]
		if !ok {
			continue
		}

		if len(depContent[4]) != 0 {
			details.replacement = depContent[3]
			details.version = depContent[4]
		} else {
			details.version = depContent[2]
		}
	}
	return nil
}

var goImportMetaRE = regexp.MustCompile(`<meta name="go-import" content="[^"\s]+ [^"\s]+ ([^"\s]+)"`)

func resolveVersions(deps map[string]*Dep) error {
	logger.Infof("Resolving versions")

	for name, dep := range deps {
		versionInfo := versionRE.FindStringSubmatch(dep.version)
		if len(versionInfo) == 0 {
			logger.Warnf("unusual version %q for %q", dep.version, name)
			continue
		}
		if len(dep.replacement) != 0 {
			name = dep.replacement
		}

		if len(versionInfo[2]) != 0 {
			dep.version = versionInfo[2]
			continue
		}

		if strings.HasPrefix(name, "github.com") {
			if split := strings.Split(name, "/"); len(split) == 4 {
				name = strings.Join(split[:3], "/")
			}
			if err := resolveVersionGH(name, versionInfo[1], dep); err != nil {
				return err
			}
			continue
		}

		resp, err := http.Get("https://" + name)
		if err != nil {
			return err
		}
		defer func() { _ = resp.Body.Close() }()

		raw, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}

		redirectInfo := goImportMetaRE.FindStringSubmatch(string(raw))
		if len(redirectInfo) == 0 {
			logger.Warnf("can't understand redirect for %q", name)
			continue
		}

		switch {
		case strings.HasPrefix(redirectInfo[1], "https://github.com"):
			if err := resolveVersionGH(strings.TrimPrefix(redirectInfo[1], "https://"), versionInfo[1], dep); err != nil {
				return err
			}
		case strings.HasPrefix(redirectInfo[1], "https://gitlab.com"):
			if err := resolveVersionGitLab(strings.TrimPrefix(redirectInfo[1], "https://"), versionInfo[1], dep); err != nil {
				return err
			}
		}
	}
	return nil
}

type GHTag struct {
	Name   string `json:"name"`
	Commit struct {
		SHA string `json:"sha"`
	} `json:"commit"`
}

func resolveVersionGH(name string, tag string, dep *Dep) error {
	logger.Debugf("Resolving version via GH for %q - %q - %+v\n", name, tag, dep)

	queryURL := fmt.Sprintf("https://api.github.com/repos/%s/tags", strings.TrimPrefix(name, "github.com/"))

	req, err := http.NewRequest(http.MethodGet, queryURL, nil)
	if err != nil {
		return err
	}
	req.Header.Set("Authorization", "Bearer "+os.Args[2])

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer func() { _ = resp.Body.Close() }()

	raw, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	parsed := []GHTag{}
	if err = json.Unmarshal(raw, &parsed); err != nil {
		return err
	}

	for _, parsedTag := range parsed {
		if parsedTag.Name == tag {
			logger.Debugf("received response for %q - %+v - %+v\n", name, parsedTag, parsedTag.Commit.SHA)
			dep.version = parsedTag.Commit.SHA[:12]
			break
		}
	}
	return nil
}

type GitLabTagResponse struct {
	Name   string `json:"name"`
	Target string `json:"target"`
}

func resolveVersionGitLab(name string, tag string, dep *Dep) error {
	logger.Debugf("Resolving version via GitLab for %q - %q - %+v\n", name, tag, dep)

	projectID := url.PathEscape(strings.TrimPrefix(name, "https://gitlab.com/"))
	queryURL := fmt.Sprintf("https://gitlab.com/api/v4/projects/%s/repository/tags/%s", projectID, tag)

	req, err := http.NewRequest(http.MethodGet, queryURL, nil)
	if err != nil {
		return err
	}
	req.Header.Set("PRIVATE-TOKEN", os.Args[3])

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer func() { _ = resp.Body.Close() }()

	raw, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	parsed := &GitLabTagResponse{}
	if err = json.Unmarshal(raw, parsed); err != nil {
		return err
	}
	logger.Debugf("received response for %q - %q - %+v\n", name, tag, parsed)
	dep.version = parsed.Target
	return nil
}