Skip to content

Instantly share code, notes, and snippets.

@ShawnMilo
Last active August 29, 2015 14:02
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save ShawnMilo/bed1bd88f53493755640 to your computer and use it in GitHub Desktop.
Save ShawnMilo/bed1bd88f53493755640 to your computer and use it in GitHub Desktop.
Convert IP to zip & country code via MaxMind (GeoLite) data in RAM, no DB Required. Proof of concept.
package main
import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"log"
"math"
"net/http"
"os"
"sort"
"strconv"
"strings"
)
// Output is for JSON marshalling.
type Output struct {
Zip string
Country string
IP string
}
// location stores the postal code and country code
type location struct {
zip string
country string
}
// record contains integers for an IPv4 address
// range and the associated zip code.
type record struct {
start int64
end int64
location
}
// recSorter is a list of record structs with the sort.Interface implemented.
// This means sort.Sort() can be called on it and it will sort by the "start" field.
type recSorter []record
func (s recSorter) Len() int { return len(s) }
func (s recSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s recSorter) Less(i, j int) bool { return s[i].start < s[j].start }
// Global slice of all records, to be sorted and queried via binary search.
var records = make(recSorter, 5e6)
// ipToInt converts a string IPv4 address to an int64.
func ipToInt(ip string) (int64, error) {
num := float64(0) // the return value
const parts = float64(3) // this minus index of dotted quad is the exponent
for i, val := range strings.Split(ip, ".") {
mult := math.Pow(float64(256), parts-float64(i))
part, err := strconv.ParseFloat(val, 64)
if err != nil {
return int64(0), err
}
num += (part * mult)
}
return int64(num), nil
}
// init loads the files into memory.
func init() {
loc, err := os.Open("GeoLiteCity-Location.csv")
if err != nil {
log.Fatal("Couldn't open location file.", err)
}
defer loc.Close()
reader := csv.NewReader(loc)
reader.FieldsPerRecord = 9
locZip := make(map[string]location, 9e5)
for {
rec, err := reader.Read()
if err != nil {
if err == io.EOF {
break
}
if strings.Contains(err.Error(), "wrong number of fields in line") {
// Skip copyright notice (invalid CSV record).
continue
}
log.Fatal(err)
}
if rec[4] == "" {
// Skip records with no zip code; waste of space.
continue
}
locZip[rec[0]] = location{rec[4], rec[1]}
}
blocks, err := os.Open("GeoLiteCity-Blocks.csv")
if err != nil {
log.Fatal("Couldn't open block file.", err)
}
defer blocks.Close()
reader = csv.NewReader(blocks)
reader.FieldsPerRecord = 3
for {
rec, err := reader.Read()
if err != nil {
if err == io.EOF {
break
}
if strings.Contains(err.Error(), "wrong number of fields in line") {
// Skip copyright notice (invalid CSV record).
continue
}
log.Fatal(err)
}
start, err := strconv.ParseInt(rec[0], 10, 64)
if err != nil {
// Skip invalid (non-digit) data. Known to happen on header record.
continue
}
end, err := strconv.ParseInt(rec[1], 10, 64)
if err != nil {
// Skip invalid (non-digit) data. Known to happen on header record.
continue
}
val := locZip[rec[2]]
if val.zip == "" {
// Don't waste our time or memory on blank records.
continue
}
records = append(records, record{start, end, val})
}
sort.Sort(records)
}
// ipToZip accepts a string IPv4 address and returns a record.
func ipToZip(ip string) record {
num, err := ipToInt(ip)
if err != nil {
// Must be an invalid zip.
return record{}
}
start := 0
end := len(records) - 1
i := 0
for {
i++
pos := ((end - start) / 2) + start
rec := records[pos]
if rec.start <= num && num <= rec.end {
return rec
}
if rec.end < num {
start = pos + 1
} else {
end = pos - 1
}
if end < start {
// Our data just doesn't have this one. Sorry.
break
}
}
return record{}
}
func main() {
http.HandleFunc("/", handler)
err := http.ListenAndServe(":2626", nil)
if err != nil {
log.Fatal(err)
}
log.Println("Listening on port 2626.")
}
// handler receives the requests and returns the responses.
func handler(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
ip := r.FormValue("ip")
if ip == "" {
ip = r.Form.Get("ip")
}
rec := ipToZip(ip)
out := Output{rec.zip, rec.country, ip}
j, err := json.Marshal(out)
var ret string
if err != nil {
ret = "{}"
log.Println(err)
} else {
ret = string(j)
}
log.Printf("%s\n", ret)
fmt.Fprintf(w, ret)
}
@ShawnMilo
Copy link
Author

I'm not convinced this is a good approach, but it was fun to try. The idea is that if you don't have a database set up, and don't need one for anything else, you can convert IPs to zip codes. If it doesn't consume too much RAM for you, then it's really convenient.

Also, to update the data you only have to download the new CSV files and restart the program.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment