unko.go

gistfile1.txt

package main

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"net/url"
)

var stock = []*url.URL{}
var base string = "http://yunocchi.org/"

func main() {
	doc, _ := goquery.NewDocument(base)
	var result []*url.URL
	doc.Find("a").Each(func(_ int, s *goquery.Selection) {
		target, _ := s.Attr("href")
		base, _ := url.Parse(base)
		targets, _ := url.Parse(target)
		result = append(result, base.ResolveReference(targets))
	})
	results := GetUrl(result)

	for {
		GetUrl(results)
		results := GetUrl(results)
		fmt.Println(results)

	}


}


func GetUrl(urls []*url.URL) []*url.URL {

 sorceUrl := []*url.URL{}
	 for _, urrrl := range urls {
		urrl := urrrl.String()
		fmt.Println(urrl)
		stock = append(stock, urrrl)
				for e := 0 ; e < len(stock) ; e++ {
					if urrrl == stock[e] {
						break 
					}
				}
				if urrl {
					break
				}
		doc, _ := goquery.NewDocument(urrl)
		var results []*url.URL
		doc.Find("a").Each(func(_ int, s *goquery.Selection) {
			target, _ := s.Attr("href")
			base, _ := url.Parse(urrl)
			targets, _ := url.Parse(target)
			results = append(results, base.ResolveReference(targets))
			sorceUrl = append(sorceUrl, results...)
		})

	}
	return sorceUrl
}