Go code for pulling all contents of a GitHub repository to the local file system. This is not a git clone, fetch, merge nor pull.

go-github.pull-all.go

package main

import (
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"

	"github.com/google/go-github/github"
)

const (
	owner    = "repo_owner"
	repo     = "repo_name"
	basePath = "/foo"
)

var client *github.Client

func main() {
	client = github.NewClient(nil)
	getContents("")
}

func getContents(path string) {
	fmt.Println("\n\n")

	fileContent, directoryContent, resp, err := client.Repositories.GetContents(owner, repo, path, nil)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Printf("%#v\n", fileContent)
	fmt.Printf("%#v\n", directoryContent)
	fmt.Printf("%#v\n", resp)

	for _, c := range directoryContent {
		fmt.Println(*c.Type, *c.Path, *c.Size, *c.SHA)

		local := filepath.Join(basePath, *c.Path)
		fmt.Println("local:", local)

		switch *c.Type {
		case "file":
			_, err := os.Stat(local)
			if err == nil {
				b, err1 := ioutil.ReadFile(local)
				if err1 == nil {
					sha := calculateGitSHA1(b)
					if *c.SHA == hex.EncodeToString(sha) {
						fmt.Println("no need to update this file, the SHA is the same")
						continue
					}
				}
			}
			downloadContents(c, local)
		case "dir":
			getContents(filepath.Join(path, *c.Path))
		}
	}
}

func downloadContents(content *github.RepositoryContent, localPath string) {
	if content.Content != nil {
		fmt.Println("content:", *content.Content)
	}

	rc, err := client.Repositories.DownloadContents(owner, repo, *content.Path, nil)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer rc.Close()

	b, err := ioutil.ReadAll(rc)
	if err != nil {
		fmt.Println(err)
		return
	}

	err = os.MkdirAll(filepath.Dir(localPath), 0666)
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println("Writing the file:", localPath)
	f, err := os.Create(localPath)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer f.Close()
	n, err := f.Write(b)
	if err != nil {
		fmt.Println(err)
	}
	if n != *content.Size {
		fmt.Printf("number of bytes differ, %d vs %d\n", n, *content.Size)
	}
}

// calculateGitSHA1 computes the github sha1 from a slice of bytes.
// The bytes are prepended with: "blob " + filesize + "\0" before runing through sha1.
func calculateGitSHA1(contents []byte) []byte {
	contentLen := len(contents)
	blobSlice := []byte("blob " + strconv.Itoa(contentLen))
	blobSlice = append(blobSlice, '\x00')
	blobSlice = append(blobSlice, contents...)
	h := sha1.New()
	h.Write(blobSlice)
	bs := h.Sum(nil)
	return bs
}

thank you very much for providing this! :) however, it's almost a decade old and some things have changed, so it doesn't work anymore. the two things that need to be changed in any case are:

• a context struct needs to be provided to client.Repositories.GetContents() and client.Repositories.DownloadContents()
• filepath.Join(path, *c.Path) needs to be replaced with *c.Path

i have created a new gist incorporating these changes as well as a few less important ones, such as error propagation, eliminating global variables, and handling (skipping) of submodules: https://gist.github.com/m1cm1c/64203ebd36ef7f39dddd053067ef3c03