phishtank_opendir.go

//get phishtank json, find links verified in the past 8 hours, remove last part of the uri, check if open dir
//first golang project
package main

import (
	"crypto/tls"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"
)

func UrlSplit(url string) (string, int) {
	urlsplit := strings.Split(url, "/")

	lastchar := url[len(url)-1:]

	if lastchar == "/" {
		urlsplit = urlsplit[:len(urlsplit)-2]
	} else {
		urlsplit = urlsplit[:len(urlsplit)-1]
	}
	urljoin := strings.Join(urlsplit, "/")
	urljoin = urljoin + "/"

	return urljoin, len(urlsplit)
}

func main() {
	var hoursold int
	flag.IntVar(&hoursold, "hoursold", 8, "Examine data after X hours ago")

	var debug bool
	flag.BoolVar(&debug, "debug", false, "Set to true to print URL's that will be checked")

	var phishtankfile string
	flag.StringVar(&phishtankfile, "phishtankfile", "", "Use phishtank json file on disk instead of connecting to phishtank")

	flag.Parse()

	timenow := time.Now()
	timeeighthoursago := timenow.Add(time.Hour * -1 * time.Duration(hoursold))

	if debug == true {
		fmt.Println(debug)
		fmt.Println("Time now", timenow)
		fmt.Println("Time specified hours ago", timeeighthoursago)
		fmt.Println("Phishtankfile", phishtankfile)
	}

	var phishtankjson []map[string]interface{}

	if len(phishtankfile) > 0 {
		content, err := ioutil.ReadFile(phishtankfile)
		if err != nil {
			fmt.Println(err)
			return
		}
		json.Unmarshal(content, &phishtankjson)
	} else {
		res, err := http.Get("http://data.phishtank.com/data/online-valid.json") //should be phishtank json URL
		if err != nil {
			fmt.Println(err)
			return
		}

		content, err := ioutil.ReadAll(res.Body)
		if err != nil {
			fmt.Println(err)
			return
		}
		json.Unmarshal(content, &phishtankjson)
	}

	var urllist []string

	for _, phishdata := range phishtankjson {
		verificationtime, err := time.Parse(time.RFC3339, fmt.Sprintf("%v", phishdata["verification_time"]))
		if err != nil {
			fmt.Println(err)
			return
		}
		if verificationtime.After(timeeighthoursago) {
			phishurl := fmt.Sprintf("%v", phishdata["url"])
			urllist = append(urllist, phishurl)
		}
	}

	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}

	var geturllist []string

	for _, value := range urllist {
		geturllist = append(geturllist, value)
		urljoin, count := UrlSplit(value)
		for count > 2 {
			geturllist = append(geturllist, urljoin)
			urljoin, count = UrlSplit(urljoin)
		}
	}

	for _, value := range geturllist {
		res, err := http.Get(value)
		if err != nil {
			fmt.Println(err)
			continue
		}
		if res.StatusCode == 200 {
			content, err := ioutil.ReadAll(res.Body)
			if err != nil {
				fmt.Println(err)
				return
			}
			htmlcontent := fmt.Sprintf("%s", content)
			htmlcontent = strings.ToLower(htmlcontent)
			if strings.Contains(htmlcontent, "index of /") {
				fmt.Printf("Open Directory found on %s\n", value)
			}
		}
	}
}