Last active
October 5, 2015 22:08
-
-
Save dustin/2885182 to your computer and use it in GitHub Desktop.
leakedin api
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// SHA Hash presence web server. | |
// | |
// Requires a file containing sorted sha1s in binary form (20 bytes each). | |
package main | |
import ( | |
"encoding/hex" | |
"fmt" | |
"io/ioutil" | |
"log" | |
"net/http" | |
"os" | |
"sort" | |
"sync/atomic" | |
"time" | |
) | |
const item_length = 20 | |
var things thingbytes | |
var processed int64 | |
type thingbytes []byte | |
type thinger interface { | |
Get(offset int) string | |
Contains(val string) bool | |
Count() int | |
} | |
func (tb thingbytes) Get(offset int) string { | |
base := offset * item_length | |
return hex.EncodeToString(tb[base : base+item_length]) | |
} | |
func (tb thingbytes) Count() int { | |
return len(tb) / item_length | |
} | |
func (tb thingbytes) Contains(needle string) bool { | |
count := tb.Count() | |
off := sort.Search(count, func(x int) bool { | |
return tb.Get(x) > needle | |
}) | |
return off < count && off > 0 && tb.Get(off-1) == needle | |
} | |
func maybefatal(err error) { | |
if err != nil { | |
log.Fatalf("Dying with %v", err) | |
} | |
} | |
func bytesReader(fn string) thingbytes { | |
b, err := ioutil.ReadFile(fn) | |
maybefatal(err) | |
log.Printf("Read all the things (%d bytes)", len(b)) | |
return thingbytes(b) | |
} | |
func searchHandler(w http.ResponseWriter, r *http.Request) { | |
p := r.URL.Path[1:] | |
atomic.AddInt64(&processed, 1) | |
w.Header().Set("Content-type", "application/json") | |
w.Header().Set("Access-Control-Allow-Origin", "*") | |
leaked := things.Contains(p) | |
if !leaked { | |
w.WriteHeader(410) | |
} | |
fmt.Fprintf(w, `{"leaked": %v}`+"\n", leaked) | |
} | |
func report() { | |
for _ = range time.Tick(time.Second) { | |
oldval := int64(0) | |
for !atomic.CompareAndSwapInt64(&processed, oldval, 0) { | |
oldval = processed | |
} | |
if oldval > 0 { | |
log.Printf("Processed %v items", oldval) | |
} | |
} | |
} | |
func main() { | |
things = bytesReader(os.Args[1]) | |
go report() | |
http.HandleFunc("/", searchHandler) | |
log.Fatal(http.ListenAndServe(":6262", nil)) | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Given a file containing sorted sha1 hashes in string form, this | |
// writes out a new file containing the same hashes in binary form. | |
package main | |
import ( | |
"bufio" | |
"encoding/hex" | |
"flag" | |
"io" | |
"log" | |
"os" | |
"github.com/dustin/go-humanize" | |
) | |
const hashSize = 20 | |
var saw int64 | |
func maybefatal(err error) { | |
if err != nil { | |
log.Fatalf("Dying with %v", err) | |
} | |
} | |
func process(r io.Reader, w io.Writer) { | |
sc := bufio.NewScanner(r) | |
for sc.Scan() { | |
bytes, err := hex.DecodeString(sc.Text()) | |
maybefatal(err) | |
n, err := w.Write(bytes) | |
maybefatal(err) | |
if n != 32 { | |
log.Fatalf("Expected to write %d bytes, wrote %d", hashSize, n) | |
} | |
saw += 1 | |
} | |
if sc.Err() != nil { | |
log.Fatalf("Error scanning: %v", sc.Err()) | |
} | |
} | |
func main() { | |
flag.Parse() | |
infile := flag.Arg(0) | |
outfile := flag.Arg(1) | |
log.Printf("%s -> %s", infile, outfile) | |
fin, err := os.Open(infile) | |
maybefatal(err) | |
defer fin.Close() | |
fout, err := os.Create(outfile) | |
maybefatal(err) | |
defer fout.Close() | |
bw := bufio.NewWriter(fout) | |
defer bw.Flush() | |
process(fin, bw) | |
log.Printf("Wrote %s items", humanize.Comma(saw)) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment