head b298f4f

vgo2nix.rewrite.patch

From e0b1531a007cce359363ac7cf701fed124fddb13 Mon Sep 17 00:00:00 2001
From: Dmitry Moskowski <me@corpix.ru>
Date: Tue, 29 Jan 2019 03:53:34 +0000
Subject: [PATCH] completely rewriting the application to add new features

---
 depsnix.go |  42 ------
 main.go    | 426 ++++++++++++++++++++++++++++++++++++++++-------------
 2 files changed, 323 insertions(+), 145 deletions(-)
 delete mode 100644 depsnix.go

diff --git a/depsnix.go b/depsnix.go
deleted file mode 100644
index d94cccd..0000000
--- a/depsnix.go
+++ /dev/null
@@ -1,42 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"os/exec"
-)
-
-func loadDepsNix() map[string]*Package {
-	ret := make(map[string]*Package)
-
-	jsonOut, err := exec.Command(
-		"nix-instantiate",
-		"--eval",
-		"--expr", "builtins.toJSON (import ./deps.nix)",
-	).Output()
-	if err != nil {
-		return ret
-	}
-
-	var layer1JSON string
-	if err := json.Unmarshal(jsonOut, &layer1JSON); err != nil {
-		panic(err)
-	}
-
-	var layer2JSON []map[string]interface{}
-	if err := json.Unmarshal([]byte(layer1JSON), &layer2JSON); err != nil {
-		panic(err)
-	}
-
-	for _, pkg := range layer2JSON {
-		goPackagePath := pkg["goPackagePath"].(string)
-		fetch := pkg["fetch"].(map[string]interface{})
-		ret[goPackagePath] = &Package{
-			GoPackagePath: goPackagePath,
-			URL: fetch["url"].(string),
-			Rev: fetch["rev"].(string),
-			Sha256: fetch["sha256"].(string),
-		}
-	}
-
-	return ret
-}
diff --git a/main.go b/main.go
index 6b57957..9a1a9ca 100644
--- a/main.go
+++ b/main.go
@@ -5,22 +5,57 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
+	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"regexp"
 	"strings"
 
 	"golang.org/x/tools/go/vcs"
 )
 
+type FlagStringArray []string
+
+func (arr *FlagStringArray) String() string {
+	return strings.Join(*arr, ", ")
+}
+
+func (arr *FlagStringArray) Set(v string) error {
+	*arr = append(*arr, v)
+	return nil
+}
+
+type Flags struct {
+	keepGoing  *bool
+	verbose    *bool
+	index      *string
+	projectDir *string
+	rewrites   *FlagStringArray
+}
+
+type App struct {
+	flags    Flags
+	rewrites map[string]string
+}
+
+type Module struct {
+	GoPackagePath string
+	Rev           string
+}
+type Modules []*Module
+
 type Package struct {
 	GoPackagePath string
 	URL           string
 	Rev           string
 	Sha256        string
 }
+type Packages []*Package
+type PackageIndex map[string]*Package
 
-const depNixFormat = `
+const dependencyTemplate = `
   {
     goPackagePath = "%s";
     fetch = {
@@ -29,147 +64,332 @@ const depNixFormat = `
       rev = "%s";
       sha256 = "%s";
     };
-  }`
+  }
+`
+
+func writeln(fd io.Writer, s string) error {
+	_, err := fd.Write([]byte(s + "\n"))
+	return err
+}
+
+func (app App) log(format string, v ...interface{}) {
+	if *app.flags.verbose {
+		log.Printf(format, v...)
+	}
+}
 
-func getPackages(keepGoing bool, prevDeps map[string]*Package) []*Package {
-	var packages []*Package
+func (app App) readDependencies(path string) (PackageIndex, error) {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		app.log("not reading dependencies from %s, file does not exists", path)
+		return PackageIndex{}, nil
+	}
 
-	commitShaRev := regexp.MustCompile(`^v\d+\.\d+\.\d+-[0-9]{14}-(.*?)$`)
-	commitRevV2 := regexp.MustCompile("^v.*-(.{12})\\+incompatible$")
-	commitRevV3 := regexp.MustCompile(`^(v\d+\.\d+\.\d+)\+incompatible$`)
+	app.log("reading dependencies from %s", path)
+	// this command will produce a string like "{\"key\": \"value\"}"
+	// we need to unwrap it
+	cmd := exec.Command(
+		"nix-instantiate",
+		"--eval",
+		"--expr",
+		fmt.Sprintf("builtins.toJSON (import %s)", path),
+	)
+	cmd.Stderr = os.Stderr
+	buf, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
 
-	cmd := exec.Command("go", "list", "-m", "all")
-	cmd.Env = append(os.Environ(),
-		"GO111MODULE=on",
+	return app.parseDependencies(buf)
+}
+
+func (app App) writeDependencies(path string, packages Packages) error {
+	fd, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+
+	err = writeln(fd, "# file generated from go.mod using vgo2nix (https://github.com/adisbladis/vgo2nix)")
+	if err != nil {
+		return err
+	}
+
+	err = writeln(fd, "[")
+	if err != nil {
+		return err
+	}
+
+	for _, pkg := range packages {
+		err = writeln(fd, fmt.Sprintf(
+			dependencyTemplate,
+			pkg.GoPackagePath, "git", pkg.URL,
+			pkg.Rev, pkg.Sha256,
+		))
+		if err != nil {
+			return err
+		}
+	}
+
+	return writeln(fd, "]")
+}
+
+func (app App) parseDependencies(buf []byte) (PackageIndex, error) {
+	app.log("parsing dependencies")
+	var (
+		packages     = make(PackageIndex)
+		unwrapped    string
+		dependencies []map[string]interface{}
 	)
-	var modList, stderr bytes.Buffer
-	cmd.Stdout = &modList
-	cmd.Stderr = &stderr
+	if err := json.Unmarshal(buf, &unwrapped); err != nil {
+		return nil, err
+	}
+
+	if err := json.Unmarshal([]byte(unwrapped), &dependencies); err != nil {
+		return nil, err
+	}
+
+	for _, pkg := range dependencies {
+		goPackagePath := pkg["goPackagePath"].(string)
+		fetch := pkg["fetch"].(map[string]interface{})
+		packages[goPackagePath] = &Package{
+			GoPackagePath: goPackagePath,
+			URL:           fetch["url"].(string),
+			Rev:           fetch["rev"].(string),
+			Sha256:        fetch["sha256"].(string),
+		}
+	}
+
+	return packages, nil
+}
+
+func (app App) readModules(root string) (Modules, error) {
+	var buf bytes.Buffer
+
+	cmd := exec.Command("go", "list", "-m", "all")
+	cmd.Dir = root
+	cmd.Env = append(os.Environ(), "GO111MODULE=on")
+	cmd.Stdout = &buf
+	cmd.Stderr = os.Stderr
+
 	err := cmd.Run()
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "'go list -m all' failed with %s:\n%s", err, stderr.String())
-		os.Exit(1)
+		return nil, err
 	}
-	// First line is always current module
-	lines := strings.Split(modList.String(), "\n")[1:]
+
+	// first line is always current module
+	return app.parseModules(strings.Split(buf.String(), "\n")[1:])
+}
+
+func (app App) parseModules(lines []string) (Modules, error) {
+	var (
+		modules     = make(Modules, len(lines))
+		commitRevV1 = regexp.MustCompile(`^v\d+\.\d+\.\d+-\d*\.?[0-9]{14}-(.*?)$`)
+		commitRevV2 = regexp.MustCompile(`^v.*-(.{12})\+incompatible$`)
+		commitRevV3 = regexp.MustCompile(`^(v\d+\.\d+\.\d+)\+incompatible$`)
+		commitRevV4 = regexp.MustCompile(`^(v\d[\d+\.]*\d)$`)
+
+		k int
+	)
 
 	for _, line := range lines {
 		if line == "" {
 			continue
 		}
 
-		l := strings.Split(line, " ")
-		var goPackagePath string
-		var revInfo string
-		if len(l) == 2 {
-			goPackagePath = l[0]
-			revInfo = l[1]
-		} else if len(l) == 5 && l[2] == "=>" {
-			goPackagePath = l[3]
-			revInfo = l[4]
-		} else {
-			panic("Wrong length")
-		}
+		var (
+			parts  = strings.Split(line, " ")
+			module = &Module{}
 
-		fmt.Println(fmt.Sprintf("Processing goPackagePath: %s", goPackagePath))
+			revInfo string
+		)
 
-		rev := revInfo
-		if commitShaRev.MatchString(rev) {
-			rev = commitShaRev.FindAllStringSubmatch(rev, -1)[0][1]
-		} else if commitRevV2.MatchString(rev) {
-			rev = commitRevV2.FindAllStringSubmatch(rev, -1)[0][1]
-		} else if commitRevV3.MatchString(rev) {
-			rev = commitRevV3.FindAllStringSubmatch(rev, -1)[0][1]
+		switch {
+		case len(parts) == 2:
+			module.GoPackagePath = parts[0]
+			revInfo = parts[1]
+		case len(parts) == 5 && parts[2] == "=>":
+			module.GoPackagePath = parts[3]
+			revInfo = parts[4]
+		default:
+			return nil, fmt.Errorf("has no parsing rules for module declaration '%s'", line)
 		}
 
-		fmt.Println(fmt.Sprintf("goPackagePath %s has rev %s", goPackagePath, rev))
-
-		var pkg *Package
-		if prevPkg, ok := prevDeps[goPackagePath]; ok {
-			if prevPkg.Rev == rev {
-				pkg = prevPkg
-			}
+		switch {
+		case commitRevV1.MatchString(revInfo):
+			module.Rev = commitRevV1.FindAllStringSubmatch(revInfo, -1)[0][1]
+		case commitRevV2.MatchString(revInfo):
+			module.Rev = commitRevV2.FindAllStringSubmatch(revInfo, -1)[0][1]
+		case commitRevV3.MatchString(revInfo):
+			module.Rev = commitRevV3.FindAllStringSubmatch(revInfo, -1)[0][1]
+		case commitRevV4.MatchString(revInfo):
+			module.Rev = commitRevV4.FindAllStringSubmatch(revInfo, -1)[0][1]
+		default:
+			return nil, fmt.Errorf("can not match rev in module declaration '%s' rev info '%s'", line, revInfo)
 		}
 
-		if pkg == nil {
-			repoRoot, err := vcs.RepoRootForImportPath(
-				goPackagePath,
-				true)
-			if err != nil {
-				panic(err)
-			}
+		modules[k] = module
+		k++
+	}
 
-			jsonOut, err := exec.Command(
-				"nix-prefetch-git",
-				"--quiet",
-				"--url", repoRoot.Repo,
-				"--rev", rev).Output()
+	return modules[:k], nil
+}
 
-			if err != nil {
-				panic(err)
-			}
-			var resp map[string]interface{}
-			if err := json.Unmarshal(jsonOut, &resp); err != nil {
-				panic(err)
-			}
-			sha256 := resp["sha256"].(string)
+func (app App) prefetchModule(module *Module, rewrites map[string]string) (*Package, error) {
+	var (
+		buf map[string]interface{}
+		url = module.GoPackagePath
+	)
+	if u, ok := rewrites[url]; ok {
+		app.log("rewriting %s to %s", module.GoPackagePath, u)
+		url = u
+	}
 
-			if sha256 == "0sjjj9z1dhilhpc8pq4154czrb79z9cm044jvn75kxcjv6v5l2m5" {
-				fmt.Println(fmt.Sprintf("Bad SHA256 for %s %s %s", goPackagePath, repoRoot.Repo, rev))
+	repoRoot, err := vcs.RepoRootForImportPath(url, *app.flags.verbose)
+	if err != nil {
+		return nil, err
+	}
 
-				if !keepGoing {
-					panic("Exiting due to bad SHA256")
-				}
-			}
+	args := []string{
+		"--url", repoRoot.Repo,
+		"--rev", module.Rev,
+	}
+	if !*app.flags.verbose {
+		args = append(args, "--quiet")
+	}
+
+	app.log("prefetching repository with nix-prefetch-git %v", args)
+	cmd := exec.Command(
+		"nix-prefetch-git",
+		args...,
+	)
+	cmd.Stderr = os.Stderr
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(out, &buf)
+	if err != nil {
+		return nil, err
+	}
+	sha256 := buf["sha256"].(string)
+
+	// XXX: this magic hash indicates problems
+	// see https://github.com/justinwoo/prefetch-github/blob/9c6dd8786d01f473569af3c6b775369f5ddfff41/prefetch-github.go#L18-L25
+	if sha256 == "0sjjj9z1dhilhpc8pq4154czrb79z9cm044jvn75kxcjv6v5l2m5" {
+		err = fmt.Errorf("bad SHA256 for package path %s, repo %s, rev %s", module.GoPackagePath, repoRoot.Repo, module.Rev)
+
+		if *app.flags.keepGoing {
+			log.Println(err)
+		} else {
+			return nil, err
+		}
+	}
+
+	return &Package{
+		GoPackagePath: module.GoPackagePath,
+		URL:           repoRoot.Repo,
+		Rev:           module.Rev,
+		Sha256:        sha256,
+	}, nil
+}
+
+func (app App) packagesFromModules(modules Modules, index PackageIndex) (Packages, error) {
+	var (
+		packages = make(Packages, len(modules))
+		rewrites = app.processRewrites([]string(*app.flags.rewrites))
+		err      error
+	)
+
+	for k, module := range modules {
+		log.Printf("processing %s %s\n", module.GoPackagePath, module.Rev)
+
+		pkg, ok := index[module.GoPackagePath]
+		if ok && pkg.Rev != module.Rev {
+			pkg = nil
+		}
 
-			pkg = &Package{
-				GoPackagePath: goPackagePath,
-				URL:           repoRoot.Repo,
-				Rev:           rev,
-				Sha256:        sha256,
+		if pkg == nil {
+			pkg, err = app.prefetchModule(module, rewrites)
+			if err != nil {
+				return nil, err
 			}
 		}
 
-		packages = append(packages, pkg)
+		packages[k] = pkg
 	}
 
-	return packages
+	return packages, nil
 }
 
-func main() {
-	var keepGoing = flag.Bool("keep-going", false, "Whether to panic or not if a rev cannot be resolved (defaults to `false`)")
-	flag.Parse()
+func (app App) processRewrites(cskv []string) map[string]string {
+	rewrites := map[string]string{}
 
-	// Load previous deps from deps.nix so we can reuse hashes for known revs
-	prevDeps := loadDepsNix()
-	packages := getPackages(*keepGoing, prevDeps)
+	for _, rewrite := range cskv {
+		fromto := strings.SplitN(rewrite, ":", 2)
+		rewrites[fromto[0]] = fromto[1]
+	}
+
+	return rewrites
+}
 
-	outfile, err := os.Create("deps.nix")
+func (app App) Run() error {
+	err := os.Chdir(*app.flags.projectDir)
 	if err != nil {
-		panic(err)
+		return err
 	}
-	defer func() {
-		if err := outfile.Close(); err != nil {
-			panic(err)
-		}
-	}()
 
-	write := func(line string) {
-		bytes := []byte(line + "\n")
-		if _, err := outfile.Write(bytes); err != nil {
-			panic(err)
-		}
+	indexPath, err := filepath.Abs(*app.flags.index)
+	if err != nil {
+		return err
+	}
+	projectDir, err := filepath.Abs(*app.flags.projectDir)
+	if err != nil {
+		return err
 	}
 
-	write("# file generated from go.mod using vgo2nix (https://github.com/adisbladis/vgo2nix)")
-	write("[")
-	for _, pkg := range packages {
-		write(fmt.Sprintf(depNixFormat,
-			pkg.GoPackagePath, "git", pkg.URL,
-			pkg.Rev, pkg.Sha256))
+	index, err := app.readDependencies(indexPath)
+	if err != nil {
+		return err
+	}
+	modules, err := app.readModules(projectDir)
+	if err != nil {
+		return err
+	}
+	packages, err := app.packagesFromModules(modules, index)
+	if err != nil {
+		return err
+	}
+	err = app.writeDependencies(indexPath, packages)
+	if err != nil {
+		return err
+	}
+
+	log.Printf("wrote %s", *app.flags.index)
+
+	return nil
+}
+
+func main() {
+	workDir, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
 	}
-	write("]")
 
-	fmt.Println("Wrote deps.nix")
+	rewrites := FlagStringArray{}
+	flag.Var(&rewrites, "rewrite", "Rewrite remote address from:to")
 
+	app := App{flags: Flags{
+		keepGoing:  flag.Bool("keep-going", false, "Whether to fail or not if a rev cannot be resolved"),
+		verbose:    flag.Bool("verbose", false, "Turn on verbose mode"),
+		index:      flag.String("index", "deps.nix", "Nix depdendencies index path"),
+		projectDir: flag.String("work-dir", workDir, "Project directory"),
+		rewrites:   &rewrites,
+	}}
+	flag.Parse()
+
+	err = app.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
 }
-- 
2.19.2