leakedin.go

// SHA Hash presence web server.
//
// Requires a file containing sorted sha1s in binary form (20 bytes each).

package main

import (
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"sort"
	"sync/atomic"
	"time"
)

const item_length = 20

var things thingbytes

var processed int64

type thingbytes []byte

type thinger interface {
	Get(offset int) string
	Contains(val string) bool
	Count() int
}

func (tb thingbytes) Get(offset int) string {
	base := offset * item_length
	return hex.EncodeToString(tb[base : base+item_length])
}

func (tb thingbytes) Count() int {
	return len(tb) / item_length
}

func (tb thingbytes) Contains(needle string) bool {
	count := tb.Count()
	off := sort.Search(count, func(x int) bool {
		return tb.Get(x) > needle
	})

	return off < count && off > 0 && tb.Get(off-1) == needle
}

func maybefatal(err error) {
	if err != nil {
		log.Fatalf("Dying with %v", err)
	}
}

func bytesReader(fn string) thingbytes {
	b, err := ioutil.ReadFile(fn)
	maybefatal(err)

	log.Printf("Read all the things (%d bytes)", len(b))

	return thingbytes(b)
}

func searchHandler(w http.ResponseWriter, r *http.Request) {
	p := r.URL.Path[1:]
	atomic.AddInt64(&processed, 1)

	w.Header().Set("Content-type", "application/json")
	w.Header().Set("Access-Control-Allow-Origin", "*")

	leaked := things.Contains(p)

	if !leaked {
		w.WriteHeader(410)
	}

	fmt.Fprintf(w, `{"leaked": %v}`+"\n", leaked)
}

func report() {
	for _ = range time.Tick(time.Second) {
		oldval := int64(0)
		for !atomic.CompareAndSwapInt64(&processed, oldval, 0) {
			oldval = processed
		}

		if oldval > 0 {
			log.Printf("Processed %v items", oldval)
		}
	}
}

func main() {
	things = bytesReader(os.Args[1])

	go report()

	http.HandleFunc("/", searchHandler)

	log.Fatal(http.ListenAndServe(":6262", nil))
}

transcode.go

// Given a file containing sorted sha1 hashes in string form, this
// writes out a new file containing the same hashes in binary form.

package main

import (
        "bufio"
        "encoding/hex"
        "flag"
        "io"
        "log"
        "os"

        "github.com/dustin/go-humanize"
)

const hashSize = 20

var saw int64

func maybefatal(err error) {
        if err != nil {
                log.Fatalf("Dying with %v", err)
        }
}

func process(r io.Reader, w io.Writer) {
        sc := bufio.NewScanner(r)
        for sc.Scan() {
                bytes, err := hex.DecodeString(sc.Text())
                maybefatal(err)

                n, err := w.Write(bytes)
                maybefatal(err)
                if n != 32 {
                        log.Fatalf("Expected to write %d bytes, wrote %d", hashSize, n)
                }

                saw += 1
        }
        if sc.Err() != nil {
                log.Fatalf("Error scanning: %v", sc.Err())
        }
}

func main() {
        flag.Parse()

        infile := flag.Arg(0)
        outfile := flag.Arg(1)

        log.Printf("%s -> %s", infile, outfile)

        fin, err := os.Open(infile)
        maybefatal(err)
        defer fin.Close()

        fout, err := os.Create(outfile)
        maybefatal(err)
        defer fout.Close()

        bw := bufio.NewWriter(fout)
        defer bw.Flush()
        process(fin, bw)
        log.Printf("Wrote %s items", humanize.Comma(saw))
}