Skip to content

Instantly share code, notes, and snippets.

@devopstaku
Forked from jasonroelofs/Timings.txt
Created July 2, 2017 13:48
Show Gist options
  • Save devopstaku/32cafd1910f93a1bc2d9bea63f4637b7 to your computer and use it in GitHub Desktop.
Save devopstaku/32cafd1910f93a1bc2d9bea63f4637b7 to your computer and use it in GitHub Desktop.
Using Go for embarrassingly parallel scripts
package main
import (
"fmt"
"net"
"io/ioutil"
"strings"
)
type DomainMap struct {
Domain string
IpMapping string
}
func retrieveDomains() []string {
file_in, _ := ioutil.ReadFile("domains.txt")
domain_list := string(file_in)
return strings.Split(strings.TrimSpace(domain_list), "\n")
}
func domainLookup(returnChannel chan DomainMap, domain string) {
rawIpAddresses, _ := net.LookupIP(domain)
// If any found, grab only the first address for simplicity as the ip lookup can return an array
ipAddress := ""
if len(rawIpAddresses) > 0 {
ipAddress = rawIpAddresses[0].String()
}
fmt.Println("Mapping: ", domain, "->", ipAddress)
// Send our results back to the main processes via our return channel
returnChannel <- DomainMap{domain, ipAddress}
}
// The extra set of parentheses here are the return type. You can give the return value a name,
// in this case +domainMapping+ and use that name in the function body. Then you don't need to specify
// what actually gets returned, you've already defined it here.
func waitForDomains(responseChannel chan DomainMap, numberOfDomains int) (domainMapping []DomainMap) {
returnedCount := 0
for {
domainMapping = append(domainMapping, <- responseChannel)
returnedCount++
if returnedCount >= numberOfDomains {
break
}
}
return
}
func main() {
domains := retrieveDomains()
// This is the channel the responses will come back on
responseChannel := make(chan DomainMap)
// Send our requests, one for each domain we get in their own goroutine
for _, domain := range domains {
go domainLookup(responseChannel, domain)
}
// Wait for all the goroutines to finish, collecting the responses
domainMapping := waitForDomains(responseChannel, len(domains))
fmt.Println(domainMapping)
}
package main
import (
"fmt"
"net"
"io/ioutil"
"strings"
)
type DomainMap struct {
Domain string
IpMapping string
}
func retrieveDomains() []string {
file_in, _ := ioutil.ReadFile("domains.txt")
domain_list := string(file_in)
return strings.Split(strings.TrimSpace(domain_list), "\n")
}
func domainLookup(domain string) DomainMap {
rawIpAddresses, _ := net.LookupIP(domain)
// If any found, grab only the first address for simplicity as the ip lookup can return an array
ipAddress := ""
if len(rawIpAddresses) > 0 {
ipAddress = rawIpAddresses[0].String()
}
fmt.Println("Mapping: ", domain, "->", ipAddress)
return DomainMap{domain, ipAddress}
}
func main() {
domains := retrieveDomains()
var domainMapping []DomainMap
// Send our requests, one for each domain we get in their
// own goroutine
for _, domain := range domains {
domainMapping = append(domainMapping, domainLookup(domain))
}
fmt.Println(domainMapping)
}
] wc -l domains.txt
783 domains.txt
] time go run domain_lookup_parallel.go
real 0m5.743s
user 0m0.359s
sys 0m0.355s
] time go run domain_lookup_sequential.go
real 0m43.794s
user 0m0.320s
sys 0m0.200s
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment