gistfile1.go

package gits

import (
	"fmt"
	"net/url"
	"os"
	"os/user"
	"runtime/debug"
	"sort"
	"strings"

	uuid "github.com/satori/go.uuid"

	"github.com/jenkins-x/jx/pkg/auth"

	"github.com/jenkins-x/jx/pkg/util"

	"github.com/jenkins-x/jx/pkg/log"

	"github.com/pkg/errors"
)

const (
	labelUpdatebot = "updatebot"
)

// EnsureUserAndEmailSetup returns the user name and email for the gitter
// lazily setting them if they are blank either from the environment variables
// `GIT_AUTHOR_NAME` and `GIT_AUTHOR_EMAIL` or using default values
func EnsureUserAndEmailSetup(gitter Gitter) (string, string, error) {
	userName, _ := gitter.Username("")
	userEmail, _ := gitter.Email("")
	if userName == "" {
		userName = os.Getenv("GIT_AUTHOR_NAME")
		if userName == "" {
			user, err := user.Current()
			if err == nil && user != nil {
				userName = user.Username
			}
		}
		if userName == "" {
			userName = "jenkins-x-bot"
		}
		err := gitter.SetUsername("", userName)
		if err != nil {
			return userName, userEmail, errors.Wrapf(err, "Failed to set the git username to %s", userName)
		}
	}
	if userEmail == "" {
		userEmail = os.Getenv("GIT_AUTHOR_EMAIL")
		if userEmail == "" {
			userEmail = "jenkins-x@googlegroups.com"
		}
		err := gitter.SetEmail("", userEmail)
		if err != nil {
			return userName, userEmail, errors.Wrapf(err, "Failed to set the git email to %s", userEmail)
		}
	}
	return userName, userEmail, nil
}

// Unshallow converts a shallow git repo (one cloned with --depth=n) into one has full depth for the current branch
// and all tags. Note that remote branches are still not fetched,
// you need to do this manually. It checks if the repo is shallow or not before trying to unshallow it.
func Unshallow(dir string, gitter Gitter) error {
	shallow, err := gitter.IsShallow(dir)
	if err != nil {
		return err
	}
	if shallow {
		if err := gitter.FetchUnshallow(dir); err != nil {
			return err
		}
		if err := gitter.FetchTags(dir); err != nil {
			return err
		}
		log.Logger().Infof("Converted %s to an unshallow repository", dir)
		return nil
	}
	return nil
}

// FetchAndMergeSHAs merges any SHAs into the baseBranch which has a tip of baseSha,
// fetching the commits from remote for the git repo in dir. It will try to fetch individual commits (
// if the remote repo supports it - see https://github.
// com/git/git/commit/68ee628932c2196742b77d2961c5e16360734a62) otherwise it uses git remote update to pull down the
// whole repo.
func FetchAndMergeSHAs(SHAs []string, baseBranch string, baseSha string, remote string, dir string,
	gitter Gitter) error {
	refspecs := make([]string, 0)
	for _, sha := range SHAs {
		refspecs = append(refspecs, fmt.Sprintf("%s:", sha))
	}
	refspecs = append(refspecs, fmt.Sprintf("%s:", baseSha))
	// First lets make sure we have the commits - remember that this may be a shallow clone
	err := gitter.FetchBranchUnshallow(dir, remote, refspecs...)
	if err != nil {
		// Unshallow fetch failed, so do a full unshallow
		// First ensure we actually have the branch refs
		err := gitter.FetchBranch(dir, remote, refspecs...)
		if err != nil {
			// This can be caused by git not being configured to allow fetching individual SHAs
			// There is not a nice way to solve this except to attempt to do a full fetch
			err = gitter.RemoteUpdate(dir)
			if err != nil {
				return errors.Wrapf(err, "updating remote %s", remote)
			}
			log.Logger().Debugf("ran %s in %s", util.ColorInfo("git remote update"), dir)
		}
		log.Logger().Debugf("ran git fetch %s %s in %s", remote, strings.Join(refspecs, " "), dir)

		err = Unshallow(dir, gitter)
		if err != nil {
			return errors.WithStack(err)
		}
		log.Logger().Debugf("Unshallowed git repo in %s", dir)
	} else {
		log.Logger().Debugf("ran git fetch --unshallow %s %s in %s", remote, strings.Join(refspecs, " "), dir)
	}
	branches, err := gitter.LocalBranches(dir)
	if err != nil {
		return errors.Wrapf(err, "listing local branches")
	}
	found := false
	for _, b := range branches {
		if b == baseBranch {
			found = true
			break
		}
	}
	if !found {
		err = gitter.CreateBranch(dir, baseBranch)
		if err != nil {
			return errors.Wrapf(err, "creating branch %s", baseBranch)
		}
	}
	// Ensure we are on baseBranch
	err = gitter.Checkout(dir, baseBranch)
	if err != nil {
		return errors.Wrapf(err, "checking out %s", baseBranch)
	}
	log.Logger().Debugf("ran git checkout %s in %s", baseBranch, dir)
	// Ensure we are on the right revision
	err = gitter.ResetHard(dir, baseSha)
	if err != nil {
		return errors.Wrapf(err, "resetting %s to %s", baseBranch, baseSha)
	}
	log.Logger().Debugf("ran git reset --hard %s in %s", baseSha, dir)
	err = gitter.CleanForce(dir, ".")
	if err != nil {
		return errors.Wrapf(err, "cleaning up the git repo")
	}
	log.Logger().Debugf("ran clean --force -d . in %s", dir)
	// Now do the merges
	for _, sha := range SHAs {
		err := gitter.Merge(dir, sha)
		if err != nil {
			return errors.Wrapf(err, "merging %s into master", sha)
		}
		log.Logger().Debugf("ran git merge %s in %s", sha, dir)

	}
	return nil
}

// SourceRepositoryProviderURL returns the git provider URL for the SourceRepository which is something like
// either `https://hostname` or `http://hostname`
func SourceRepositoryProviderURL(gitProvider GitProvider) string {
	return GitProviderURL(gitProvider.ServerURL())
}

// GitProviderURL returns the git provider host URL for the SourceRepository which is something like
// either `https://hostname` or `http://hostname`
func GitProviderURL(text string) string {
	if text == "" {
		return text
	}
	u, err := url.Parse(text)
	if err != nil {
		log.Logger().Warnf("failed to parse git provider URL %s: %s", text, err.Error())
		return text
	}
	u.Path = ""
	if !strings.HasPrefix(u.Scheme, "http") {
		// lets convert other schemes like 'git' to 'https'
		u.Scheme = "https"
	}
	return u.String()
}

// PushRepoAndCreatePullRequest commits and pushes the changes in the repo rooted at dir.
// It creates a branch called branchName from a base.
// It uses the pullRequestDetails for the message and title for the commit and PR.
// It uses and updates pullRequestInfo to identify whether to rebase an existing PR.
func PushRepoAndCreatePullRequest(dir string, gitInfo *GitRepository, base string, prDetails *PullRequestDetails, filter *PullRequestFilter, commit bool, commitMessage string, push bool, autoMerge bool, dryRun bool, gitter Gitter, provider GitProvider) (*PullRequestInfo, error) {
	if commit {

		err := gitter.Add(dir, "-A")
		if err != nil {
			return nil, errors.WithStack(err)
		}
		changed, err := gitter.HasChanges(dir)
		if err != nil {
			return nil, errors.WithStack(err)
		}
		if !changed {
			log.Logger().Warnf("No changes made to the source code in %s. Code must be up to date!", dir)
			return nil, nil
		}
		if commitMessage == "" {
			commitMessage = prDetails.Message
		}
		err = gitter.CommitDir(dir, commitMessage)
		if err != nil {
			return nil, errors.WithStack(err)
		}
	}

	headPrefix := ""

	username := provider.CurrentUsername()
	if username == "" {
		return nil, fmt.Errorf("no git user name found")
	}
	if gitInfo.Organisation != username && gitInfo.Fork {
		headPrefix = username + ":"
	}

	gha := &GitPullRequestArguments{
		GitRepository: gitInfo,
		Title:         prDetails.Title,
		Body:          prDetails.Message,
		Base:          base,
	}

	if filter != nil && push {

		// lets rebase an existing PR
		existingPrs, err := FilterOpenPullRequests(provider, gitInfo.Organisation, gitInfo.Name, *filter)
		if err != nil {
			return nil, errors.Wrapf(err, "finding existing PRs using filter %s on repo %s/%s", filter.String(), gitInfo.Organisation, gitInfo.Name)
		}

		if len(existingPrs) > 1 {
			sort.SliceStable(existingPrs, func(i, j int) bool {
				// sort in descending order of PR numbers (assumes PRs numbers increment!)
				return util.DereferenceInt(existingPrs[j].Number) < util.DereferenceInt(existingPrs[i].Number)
			})
			prs := make([]string, 0)
			for _, pr := range existingPrs {
				prs = append(prs, pr.URL)
			}
			log.Logger().Debugf("Found more than one PR %s using filter %s on repo %s/%s so rebasing latest PR %s", strings.Join(prs, ", "), filter.String(), gitInfo.Organisation, gitInfo.Name, existingPrs[:1][0].URL)
			//
			existingPrs = existingPrs[:1]
		}
		if len(existingPrs) == 1 {
			pr := existingPrs[0]
			// We can only update an existing PR if the owner of that PR is this user!
			if util.DereferenceString(pr.HeadOwner) == username && pr.HeadRef != nil && pr.Number != nil {
				remote := "origin"
				if gitInfo.Fork {
					remote = "upstream"
				}
				url := pr.URL
				changeBranch, err := gitter.Branch(dir)
				if err != nil {
					return nil, errors.WithStack(err)
				}
				localBranchUUID, err := uuid.NewV4()
				if err != nil {
					return nil, errors.Wrapf(err, "creating UUID for local branch")
				}
				// We use this "dummy" local branch to pull into to avoid having to work with FETCH_HEAD as our local
				// representation of the remote branch. This is an oddity of the pull/%d/head remote.
				localBranch := localBranchUUID.String()
				existingBranchName := *pr.HeadRef
				fetchRefSpec := fmt.Sprintf("pull/%d/head:%s", *pr.Number, localBranch)
				err = gitter.FetchBranch(dir, remote, fetchRefSpec)
				if err != nil {
					return nil, errors.Wrapf(err, "fetching %s for merge", fetchRefSpec)
				}

				err = gitter.CreateBranchFrom(dir, prDetails.BranchName, localBranch)
				if err != nil {
					return nil, errors.Wrapf(err, "creating branch %s from %s", prDetails.BranchName, fetchRefSpec)
				}
				err = gitter.Checkout(dir, prDetails.BranchName)
				if err != nil {
					return nil, errors.Wrapf(err, "checking out branch %s", prDetails.BranchName)
				}
				err = gitter.MergeTheirs(dir, changeBranch)
				if err != nil {
					return nil, errors.Wrapf(err, "merging %s into %s", changeBranch, fetchRefSpec)
				}
				err = gitter.RebaseTheirs(dir, fmt.Sprintf(localBranch), "", true)
				if err != nil {
					return nil, errors.WithStack(err)
				}
				if dryRun {
					log.Logger().Infof("Commit created but not pushed; would have updated pull request %s with %s and used commit message %s. Please manually delete %s when you are done", util.ColorInfo(pr.URL), prDetails.String(), commitMessage, util.ColorInfo(dir))
					return nil, nil
				}
				err = gitter.ForcePushBranch(dir, prDetails.BranchName, existingBranchName)
				if err != nil {
					return nil, errors.Wrapf(err, "pushing merged branch %s", existingBranchName)
				}

				gha.Head = headPrefix + existingBranchName
				// work out the minimal similar title
				if strings.HasPrefix(pr.Title, "chore(deps): bump ") {
					origWords := strings.Split(pr.Title, " ")
					newWords := strings.Split(prDetails.Title, " ")
					answer := make([]string, 0)
					for i, w := range newWords {
						if len(origWords) > i && origWords[i] == w {
							answer = append(answer, w)
						}
					}
					if answer[len(answer)-1] == "bump" {
						// if there are no similarities in the actual dependency, then add a generic form of words
						answer = append(answer, "dependency", "versions")
					}
					if answer[len(answer)-1] == "to" || answer[len(answer)-1] == "from" {
						// remove trailing prepositions
						answer = answer[:len(answer)-1]
					}
					gha.Title = strings.Join(answer, " ")
				} else {
					gha.Title = prDetails.Title
				}
				gha.Body = fmt.Sprintf("%s\n<hr />\n\n%s", prDetails.Message, pr.Body)

				pr, err := provider.UpdatePullRequest(gha, *pr.Number)
				if err != nil {
					return nil, errors.Wrapf(err, "updating pull request %s", url)
				}
				log.Logger().Infof("Updated Pull Request: %s", util.ColorInfo(pr.URL))
				return &PullRequestInfo{
					GitProvider:          provider,
					PullRequest:          pr,
					PullRequestArguments: gha,
				}, nil
			}
		} else {
			log.Logger().Debugf("Did not find any PRs to rebase, creating a new one")
		}
	}

	gha.Head = headPrefix + prDetails.BranchName

	if dryRun {
		log.Logger().Infof("Commit created but not pushed; would have created new pull request with %s and used commit message %s. Please manually delete %s when you are done.", prDetails.String(), commitMessage, util.ColorInfo(dir))
		return nil, nil
	}

	if push {
		err := gitter.ForcePushBranch(dir, "HEAD", prDetails.BranchName)
		if err != nil {
			return nil, err
		}
	}

	pr, err := provider.CreatePullRequest(gha)
	if err != nil {
		return nil, errors.Wrapf(err, "creating pull request with arguments %v", gha.String())
	}
	log.Logger().Infof("Created Pull Request: %s", util.ColorInfo(pr.URL))
	if autoMerge {
		number := *pr.Number
		err = provider.AddLabelsToIssue(pr.Owner, pr.Repo, number, []string{labelUpdatebot})
		if err != nil {
			return nil, err
		}
		log.Logger().Infof("Added label %s to Pull Request %s", util.ColorInfo(labelUpdatebot), pr.URL)
	}
	return &PullRequestInfo{
		GitProvider:          provider,
		PullRequest:          pr,
		PullRequestArguments: gha,
	}, nil
}

// ForkAndPullPullRepo pulls the specified gitUrl into dir, creating a remote fork if needed using the git provider
// It is the callers responsibility to create a tempory dir if necessary
func ForkAndPullPullRepo(gitURL string, dir string, baseRef string, branchName string, provider GitProvider, gitter Gitter, configGitFn ConfigureGitFn) (string, string, *GitRepository, error) {
	if gitURL == "" {
		return "", "", nil, fmt.Errorf("No source gitter URL")
	}
	gitInfo, err := ParseGitURL(gitURL)
	gitInfo.Fork = false
	if err != nil {
		return "", "", nil, errors.Wrapf(err, "failed to parse gitter URL %s", gitURL)
	}

	username := ""
	userDetails := auth.UserAuth{}
	originalOrg := gitInfo.Organisation
	originalRepo := gitInfo.Name

	if provider == nil {
		log.Logger().Warnf("No GitProvider specified!")
		debug.PrintStack()
	} else {
		userDetails = provider.UserAuth()
		username = provider.CurrentUsername()

		// lets check if we need to fork the repository...
		if originalOrg != username && username != "" && originalOrg != "" && provider.ShouldForkForPullRequest(originalOrg, originalRepo, username) {
			gitInfo.Fork = true
		}
	}
	if baseRef == "" {
		baseRef = "master"
	}

	if gitInfo.Fork {
		if provider == nil {
			return "", "", nil, errors.Wrapf(err, "no Git Provider specified for gitter URL %s", gitURL)
		}
		repo, err := provider.GetRepository(username, originalRepo)
		if err != nil {
			// lets try create a fork - using a blank organisation to force a user specific fork
			repo, err = provider.ForkRepository(originalOrg, originalRepo, "")
			if err != nil {
				return "", "", nil, errors.Wrapf(err, "failed to fork GitHub repo %s/%s to user %s", originalOrg, originalRepo, username)
			}
			log.Logger().Infof("Forked Git repository to %s\n", util.ColorInfo(repo.HTMLURL))
		}

		// lets only use this repository if it is a fork
		if !repo.Fork {
			gitInfo.Fork = false
		} else {
			cloneGitURL, err := gitter.CreatePushURL(repo.CloneURL, &userDetails)
			if err != nil {
				return "", "", nil, errors.Wrapf(err, "failed to get clone URL from %s and user %s", repo.CloneURL, username)
			}
			err = gitter.Clone(cloneGitURL, dir)
			if err != nil {
				return "", "", nil, errors.WithStack(err)
			}
			err = gitter.FetchBranch(dir, "origin")
			if err != nil {
				return "", "", nil, errors.Wrapf(err, "fetching from %s", cloneGitURL)
			}
			err = gitter.SetRemoteURL(dir, "upstream", gitURL)
			if err != nil {
				return "", "", nil, errors.Wrapf(err, "setting remote upstream %q in forked environment repo", gitURL)
			}
			if configGitFn != nil {
				err = configGitFn(dir, gitInfo, gitter)
				if err != nil {
					return "", "", nil, errors.WithStack(err)
				}
			}
			branchName, err := computeBranchName(baseRef, branchName, dir, gitter)
			if err != nil {
				return "", "", nil, errors.WithStack(err)
			}
			if branchName != "master" {
				err = gitter.CreateBranch(dir, branchName)
				if err != nil {
					return "", "", nil, errors.WithStack(err)
				}

				err = gitter.Checkout(dir, branchName)
				if err != nil {
					return "", "", nil, errors.WithStack(err)
				}
			}
			err = gitter.ResetToUpstream(dir, baseRef)
			if err != nil {
				return "", "", nil, errors.Wrapf(err, "resetting forked branch %s to upstream version", baseRef)
			}
			return dir, baseRef, gitInfo, nil
		}
	}

	// now lets clone the fork and pull it...
	exists, err := util.FileExists(dir)
	if err != nil {
		return "", "", nil, errors.Wrapf(err, "failed to check if directory %s exists", dir)
	}

	if exists {
		if configGitFn != nil {
			err = configGitFn(dir, gitInfo, gitter)
			if err != nil {
				return "", "", nil, errors.WithStack(err)
			}
		}
		// lets check the gitter remote URL is setup correctly
		err = gitter.SetRemoteURL(dir, "origin", gitURL)
		if err != nil {
			return "", "", nil, errors.WithStack(err)
		}
		err = gitter.Stash(dir)
		if err != nil {
			return "", "", nil, errors.WithStack(err)
		}
		branchName, err := computeBranchName(baseRef, branchName, dir, gitter)
		if err != nil {
			return "", "", nil, errors.WithStack(err)
		}
		if branchName != "master" {
			err = gitter.CreateBranch(dir, branchName)
			if err != nil {
				return "", "", nil, errors.WithStack(err)
			}
		}
		err = gitter.Checkout(dir, branchName)
		if err != nil {
			return "", "", nil, errors.WithStack(err)
		}
		err = gitter.FetchBranch(dir, "origin", baseRef)
		if err != nil {
			return "", "", nil, errors.WithStack(err)
		}
		err = gitter.Merge(dir, baseRef)
		if err != nil {
			return "", "", nil, errors.WithStack(err)
		}
	} else {
		err := os.MkdirAll(dir, util.DefaultWritePermissions)
		if err != nil {
			return "", "", nil, fmt.Errorf("failed to create directory %s due to %s", dir, err)
		}
		cloneGitURL, err := gitter.CreatePushURL(gitURL, &userDetails)
		if err != nil {
			return "", "", nil, errors.Wrapf(err, "failed to get clone URL from %s and user %s", gitURL, username)
		}

		err = gitter.Clone(cloneGitURL, dir)
		if err != nil {
			return "", "", nil, errors.WithStack(err)
		}
		if configGitFn != nil {
			err = configGitFn(dir, gitInfo, gitter)
			if err != nil {
				return "", "", nil, errors.WithStack(err)
			}
		}
		branchName, err := computeBranchName(baseRef, branchName, dir, gitter)
		if err != nil {
			return "", "", nil, errors.WithStack(err)
		}
		if branchName != "master" {
			err = gitter.CreateBranch(dir, branchName)
			if err != nil {
				return "", "", nil, errors.WithStack(err)
			}

			err = gitter.Checkout(dir, branchName)
			if err != nil {
				return "", "", nil, errors.WithStack(err)
			}
		}
	}
	return dir, baseRef, gitInfo, nil
}

// A PullRequestFilter defines a filter for finding pull requests
type PullRequestFilter struct {
	Labels []string
	Number *int
}

func (f *PullRequestFilter) String() string {
	return strings.Join(f.Labels, ", ")
}

// FilterOpenPullRequests looks for any pull requests on the owner/repo where all the labels match
func FilterOpenPullRequests(provider GitProvider, owner string, repo string, filter PullRequestFilter) ([]*GitPullRequest, error) {
	openPRs, err := provider.ListOpenPullRequests(owner, repo)
	if err != nil {
		return nil, errors.Wrapf(err, "listing open pull requests on %s/%s", owner, repo)
	}
	answer := make([]*GitPullRequest, 0)
	for _, pr := range openPRs {
		if len(filter.Labels) > 0 {
			found := 0
			for _, label := range filter.Labels {
				f := false
				for _, prLabel := range pr.Labels {
					if label == util.DereferenceString(prLabel.Name) {
						f = true
					}
				}
				if f {
					found++
				}
			}
			if len(filter.Labels) == found {
				answer = append(answer, pr)
			}
		}
		if filter.Number != nil && filter.Number == pr.Number {
			answer = append(answer, pr)
		}
	}
	return answer, nil
}

func computeBranchName(baseRef string, branchName string, dir string, gitter Gitter) (string, error) {

	if branchName == "" {
		branchName = baseRef
	}
	validBranchName := gitter.ConvertToValidBranchName(branchName)

	branchNames, err := gitter.RemoteBranchNames(dir, "remotes/origin/")
	if err != nil {
		return "", errors.Wrapf(err, "Failed to load remote branch names")
	}
	if util.StringArrayIndex(branchNames, validBranchName) >= 0 {
		// lets append a UUID as the branch name already exists
		branchNameUUID, err := uuid.NewV4()
		if err != nil {
			return "", errors.WithStack(err)
		}
		validBranchName += "-" + branchNameUUID.String()
	}
	return validBranchName, nil
}

//IsUnadvertisedObjectError returns true if the reason for the error is that the request was for an object that is unadvertised (i.e. doesn't exist)
func IsUnadvertisedObjectError(err error) bool {
	return strings.Contains(err.Error(), "Server does not allow request for unadvertised object")
}

func parseAuthor(l string) (string, string) {
	open := strings.Index(l, "<")
	close := strings.Index(l, ">")
	return strings.TrimSpace(l[:open]), strings.TrimSpace(l[open+1 : close])
}