crawl repository for internal godoc

crawler.go

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
)

var (
	client *http.Client

	oAuthToken  = ""
	githubURL   = "https://api.github.com"
	gddoBaseURL = "http://127.0.0.1:8080"
	// gddoBaseURL = "http://127.0.0.1:8080"

	owner = "[GROUP OR REPOSITORY OWNER NAME]"

	repoURL           = "github.com/" + owner + "/%s"
	repositoryListURL = "/orgs/" + owner + "/repos?per_page=1000"
	repoContentURL    = "/repos/" + owner + "/%s/contents/%s"
)

func checkErr(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

type Repo struct {
	Name string `json:"name"`
}

type Content struct {
	Type string `json:"type"`
	Name string `json:"name"`
}

func init() {
	client = &http.Client{}
}

func main() {
	req, err := http.NewRequest("GET", githubURL+repositoryListURL, nil)
	checkErr(err)
	req.Header.Add("Authorization", fmt.Sprintf("token %s", oAuthToken))

	resp, err := client.Do(req)
	checkErr(err)

	body, err := ioutil.ReadAll(resp.Body)
	checkErr(err)

	var listRepo []Repo
	json.Unmarshal(body, &listRepo)

	for _, repo := range listRepo {
		reqURL := fmt.Sprintf(repoContentURL, repo.Name, "src")

		listContent := getContents(reqURL)

		hitURL := fmt.Sprintf("%s/github.com/%s/%s", gddoBaseURL, owner, repo.Name)
		http.Get(hitURL)

		if len(listContent) > 0 {
			for _, pkg := range listContent {
				if pkg.Type == "dir" {
					resp := hitGodoc(fmt.Sprintf("%s/src/%s", repo.Name, pkg.Name))
					if resp.StatusCode != 200 {
						reqURL := fmt.Sprintf(repoContentURL, repo.Name, "src/"+pkg.Name)
						listContent := getContents(reqURL)
						for _, pkgd := range listContent {
							if pkgd.Type == "dir" {
								hitGodoc(fmt.Sprintf("%s/src/%s/%s", repo.Name, pkg.Name, pkgd.Name))
							}
						}
					}
				}
			}
		}

	}
}

func hitGodoc(path string) *http.Response {
	hitURL := fmt.Sprintf("%s/github.com/%s/%s", gddoBaseURL, owner, path)
	resp, err := http.Get(hitURL)
	if err != nil {
		log.Println(" ERR: ", err.Error())
	}
	if resp.StatusCode != 200 {
		log.Println(resp.StatusCode, " - ", hitURL)
	}
	return resp
}

func getContents(reqURL string) []Content {
	req, err := http.NewRequest("GET", githubURL+reqURL, nil)
	checkErr(err)
	req.Header.Add("Authorization", fmt.Sprintf("token %s", oAuthToken))

	resp, err := client.Do(req)
	checkErr(err)

	body, err := ioutil.ReadAll(resp.Body)
	checkErr(err)

	var listContent []Content
	json.Unmarshal(body, &listContent)
	return listContent
}