crtsh-lookup.go

package main

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"
)

type crtsh struct {
	MinCertID int `json:"min_cert_id"`
}

var noCertDomains []string
var links []string

func main() {
	file, err := os.Open("search-domains.csv")
	if err != nil {
		log.Fatal("Couldn't open the file to parse")
	}
	domainNames := csv.NewReader(file)

	t := &http.Transport{
		MaxIdleConns:      100,
		IdleConnTimeout:   1,
		DisableKeepAlives: true,
	}
	client := &http.Client{
		Timeout:   time.Second * 2,
		Transport: t,
	}

	count := 0
	c := make(chan string, 100)
	var wg sync.WaitGroup
	go func(client *http.Client, wg *sync.WaitGroup) {
		for domainName := range c {
			count = count + 1
			/*if count%20 == 0 {
				println("open files:", countOpenFiles())
			}*/
			println("Processing", count, domainName, len(c))
			go queryCrtsh(domainName, wg, client)
		}
	}(client, &wg)

	countAdd := 0
	for {
		domainName, err := domainNames.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}
		countAdd = countAdd + 1
		if countAdd%100 == 0 {
			println(countAdd, "-- Adding to the channel:", domainName[0])
		}
		wg.Add(1)
		c <- domainName[0]
	}

	close(c)
	wg.Wait()

	fmt.Print(strings.Join(links[:], "\n"))
	println("Found %d links", len(links))
	println("Unfound domains %d", len(noCertDomains))
}

func queryCrtsh(domainName string, wg *sync.WaitGroup, client *http.Client) {
	defer wg.Done()

	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://crt.sh/?q=%s&output=json", domainName), nil)
	if err != nil {
		println(domainName)
		log.Println(fmt.Sprintf("Failed to create request"))
		log.Fatal(err)
	}
	req.Close = true

	req.Header.Set("User-Agent", "Certopia")
	req.Header.Set("Connection", "close")
	res, err := client.Do(req)
	if err != nil {
		println(domainName)
		log.Println("Failed to get response", domainName, countOpenFiles())
		log.Fatal(err)
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		println(domainName)
		log.Println(fmt.Sprintf("Failed to read response"))
		log.Fatal(err)
	}

	formatBody := fmt.Sprintf("[%s]", strings.Replace(string(body[:]), "}{", "},{", -1))
	var certs []crtsh
	err = json.Unmarshal([]byte(formatBody), &certs)
	if err != nil {
		println(domainName)
		log.Println(fmt.Sprintf("Failed to unmarshal body %s", formatBody))
		log.Fatal(err)
	}

	if len(certs) == 0 {

		noCertDomains = append(noCertDomains, domainName)
	}

	// for loop gets a list of certs, so pre and leaf
	for i := range certs {
		links = append(links, fmt.Sprintf("https://crt.sh/?id=%d", certs[i].MinCertID))
	}
}

func countOpenFiles() int {
	out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("lsof -p %v", os.Getpid())).Output()
	if err != nil {
		log.Fatal(err)
	}
	lines := strings.Split(string(out), "\n")
	return len(lines) - 1
}