Skip to content

Instantly share code, notes, and snippets.

@nathan-osman
Last active February 2, 2017 09:14
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save nathan-osman/ef34385ea2c8f132bcf37d60c735e7e2 to your computer and use it in GitHub Desktop.
Save nathan-osman/ef34385ea2c8f132bcf37d60c735e7e2 to your computer and use it in GitHub Desktop.
Test a list of domains
package main
import (
"bufio"
"log"
"net/http"
"net/url"
"os"
"sync"
"time"
)
const (
// Change this to the number of concurrent requests you want
NUM_GOROUTINES = 16
// Number of seconds before giving up on a request
READ_TIMEOUT = 10
)
// Used for synchronizing the goroutines.
var waitGroup sync.WaitGroup
// processUrl continuously reads URLs from the channel and tests them until the
// channel is closed.
func processUrl(c <-chan string) {
// When the function returns, decrement the WaitGroup
defer waitGroup.Done()
for rawUrl := range c {
// Ensure the URL is of the form "http://..."
u, err := url.Parse(rawUrl)
if err != nil {
log.Printf("%s:\t%s", rawUrl, err.Error())
return
}
u.Scheme = "http"
// Send the request
r, err := http.Get(u.String())
if err != nil {
log.Printf("%s:\t%s", u, err.Error())
return
}
// Print the result
log.Printf("%s:\t%s", u, r.Status)
}
}
func main() {
var (
scanner = bufio.NewScanner(os.Stdin)
urlChan = make(chan string)
)
// Ensure requests don't get stuck
http.DefaultClient.Timeout = READ_TIMEOUT * time.Second
// Spawn x number of goroutines
waitGroup.Add(NUM_GOROUTINES)
for i := 0; i < NUM_GOROUTINES; i++ {
go processUrl(urlChan)
}
// Feed the lines into the goroutines (skipping blank lines)
for scanner.Scan() {
t := scanner.Text()
if len(t) > 0 {
urlChan <- t
}
}
// Close the channel to indicate the end of the data
close(urlChan)
// Wait for all the goroutines to finish
waitGroup.Wait()
}
@JafferWilson
Copy link

Thank you.. I think this will work.. Checking...:)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment