clawler.go

package main

import (
	"fmt"
	"net/url"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

var stock = []string{}
var base string = "https://github.com/7yan00"

func main() {
	result := makeUrl(base)
	results := GetUrl(result)

	for len(results) > 0 {
		results = GetUrl(results)
	}

	fmt.Println(results)

}

func GetUrl(urls []*url.URL) []*url.URL {

	sorceUrl := []*url.URL{}
L:
	for _, urrrl := range urls {
		urrl := urrrl.String()
		for e := 0; e < len(stock); e++ {
			if urrl == stock[e] {
				continue L
			}
		}
		if !strings.Contains(urrl, base) {
			continue L
		}
		fmt.Println(urrrl)
		stock = append(stock, urrl)
		/*	doc, _ := goquery.NewDocument(urrl)
			var results []*url.URL
			doc.Find("a").Each(func(_ int, s *goquery.Selection) {
				target, _ := s.Attr("href")
				base, _ := url.Parse(urrl)
				targets, _ := url.Parse(target)
				results = append(results, base.ResolveReference(targets)) */
		results := makeUrl(urrl)
		sorceUrl = append(sorceUrl, results...)
	}
	return sorceUrl
}

func makeUrl(base string) []*url.URL {
	doc, _ := goquery.NewDocument(base)
	var result []*url.URL
	doc.Find("a").Each(func(_ int, s *goquery.Selection) {
		target, _ := s.Attr("href")
		base, _ := url.Parse(base)
		targets, _ := url.Parse(target)
		result = append(result, base.ResolveReference(targets))
	})
	return result
}