golang module to handle BPjM Lists

bpjm.go

// go library to scope with the BPjM Censorship list

package Bpjm

import (
	"net/url"
	"regexp"
	"crypto/md5"
	"strings"
	"bytes"
	"io/ioutil"
	"encoding/binary"
	"io"
)

// BpjmEntry is a datastructure for BPjM List Entries
type BpjmEntry struct {
	DomainMd5 []byte // Md5 sum of domain, with a leading http
	PathMd5   []byte
	Depth     int32
}

// parseURL - takes an URL string as input and returns a pointer to an url.URL object
func ParseURL(ustr string) (u *url.URL, err error) {
	u, err = url.Parse(ustr)
	return u, err
}

// XtrctHost - takes a url object and returns the host
func XtrctHost(u *url.URL) (host string) {
	return u.Host
}

// XtrctPath  - takes an url object and returns the path
func XtrctPath(u *url.URL) (path string) {
	return u.Path
}

// FilterHost - Filters host according to BPjM Host Filters
//  effectivley it cuts away a leading 'www.' subdomain, if existing
func FilterHost(host string) (fhost string) {
	re := regexp.MustCompile("^www.")
	fhost = re.ReplaceAllString(host, "")
	return fhost
}

// FilterPath - Filters path according to BPjM Path Filters
//  effectivley cuts away the root directory from path, if existing
func FilterPath(path string) (fpath string) {
	re := regexp.MustCompile("^/")
	fpath = re.ReplaceAllString(path, "")
	return fpath
}

// GenMd5 - Generates n MD5 checksum of a given string and returns the md5sum as array of bytes
func GenMd5(in string) (md5sum []byte) {
	inb := []byte(in)
	md5 := md5.Sum(inb)
	for _, c := range md5 {
		md5sum = append(md5sum, c)
	}
	return md5sum
}

// check - is a general error checking routine that panics on error
func check(err error) {
	if err != nil {
		panic(err)
	}
}

// enumerate depth of a given URL
func EnumerateDepth(u *url.URL) (depth int32) {
	path := XtrctPath(u)
	i := len(strings.Split(path, "/"))
	return int32(i)
}

// UrlToBpjmEntry - takes an URL string as input and returns a BpjmEntry object
func UrlToBpjmEntry(ustr string) (entry BpjmEntry) {
	u, err := ParseURL(ustr)
	check(err)
	host := FilterHost( XtrctHost(u) )
	path := FilterPath( XtrctPath(u) )
	depth := EnumerateDepth(u)
	e := BpjmEntry{
		GenMd5("http://" + host),
		GenMd5(path),
		depth,
	}
	return e
}

// Load a binary BPjM ListFile from a Fritzbox
func LoadFritzBoxFile(filename string)(data []byte) {
	data, _ = ioutil.ReadFile(filename)
	return data
}

// FritzBoxFile - datastructure to describe an Fritzbox BPjM File
type FritzBoxFile struct {
	Magick			[]byte
	EmbedFileName	string
	Size			int
	Records			int
	Entries			[]BpjmEntry
}

func ParseFritzBoxFile(data []byte)(FBF FritzBoxFile){
	// set up a buffer we can work with
	buffer := bytes.NewBuffer(data)
	// get magick; first 5 bytes
	magick := buffer.Next(5)
	// get embedded filename; next 59 bytes; cut out the null bytes and make it a string
	embedFileName := string( bytes.Trim(buffer.Next(59), string(0x00)) )
	// get size of file in bytes
	size := buffer.Len()
	// prepager a list to store entries
	var Entries []BpjmEntry
	// init counter
	n := 0
	// for each entry in list...
	for {
		// ... read domain hash
		d := make([]byte, 16)
		_, err := buffer.Read(d)
		if err != nil {
			if err == io.EOF {
				break
			}
			panic(err)
		}
		// ...read path hash
		p := make([]byte, 16)
		_, err = buffer.Read(p)
		if err != nil {
			if err == io.EOF {
				break
			}
			panic(err)
		}
		//...read depth
		i, err := binary.ReadUvarint(buffer)
		if err != nil {
			if err == io.EOF {
				break
			}
			panic(err)
		}
		// parse read data into an BpjmEntry object
		e := BpjmEntry{
			d,
			p,
			int32(i),
		}
		// append object to list of objects
		Entries = append(Entries, e)
		// count
		n++
	}
	// parse collected data into a FirtzBoxFile Object
	FBF = FritzBoxFile{
		magick,
		embedFileName,
		size,
		n,
		Entries,
	}
	// return the FBF Object
	return FBF
}

for a prog that uses this module see: https://gist.github.com/scusi/ce9b67b4a559cd7eb3f9