hogehoge.go

package main

import (
	"fmt"
	"net/url"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

var stock = []string{}
var base string = "http://yunocchi.org/"

func main() {
	doc, _ := goquery.NewDocument(base)
	var result []*url.URL
	doc.Find("a").Each(func(_ int, s *goquery.Selection) {
		target, _ := s.Attr("href")
		base, _ := url.Parse(base)
		targets, _ := url.Parse(target)
		result = append(result, base.ResolveReference(targets))
	})
	results := GetUrl(result)
	fmt.Println(results)

	for len(results) > 0 {
		results = GetUrl(results)
		fmt.Println(results)
	}

}

func GetUrl(urls []*url.URL) []*url.URL {

	sorceUrl := []*url.URL{}
L:
	for _, urrrl := range urls {
		urrl := urrrl.String()
		fmt.Println(urrl)
		for e := 0; e < len(stock); e++ {
			if urrl == stock[e] {
				break L
			}
		}
		if !strings.Contains(urrl, base) {
			break
		}
		stock = append(stock, urrl)
		doc, _ := goquery.NewDocument(urrl)
		var results []*url.URL
		doc.Find("a").Each(func(_ int, s *goquery.Selection) {
			target, _ := s.Attr("href")
			base, _ := url.Parse(urrl)
			targets, _ := url.Parse(target)
			results = append(results, base.ResolveReference(targets))
			sorceUrl = append(sorceUrl, results...)
		})

	}
	return sorceUrl
}