Skip to content

Instantly share code, notes, and snippets.

@White2001Offl
Created June 6, 2021 05:13
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save White2001Offl/443080b0afba48bcb8508efca9ab41de to your computer and use it in GitHub Desktop.
Save White2001Offl/443080b0afba48bcb8508efca9ab41de to your computer and use it in GitHub Desktop.
ProxyPool in go with thread safe
/*Its a simple implementation where follows round robin format to access proxies in thread or normal method*/
/*The proxies should be loaded in format ip:port:user:pass*/
import (
"sync"
"fmt"
"os"
"bufio"
"strings"
)
var poolMutex = &sync.Mutex{} // Should be in global scope
type proxyPool struct{ // A Custom type for ProxyPool where stores the current count and proxies
proxy *[]string
count int
}
func prepProxy (proxy string) string{ // Returns the proxy which is compatible in GO Url Parse
splitter := strings.Split(proxy,":")
if len(splitter) == 2{
return fmt.Sprintf("http://%v:%v",splitter[0],splitter[1])
} else{
return fmt.Sprintf("http://%v:%v@%v:%v",splitter[2],splitter[3],splitter[0],splitter[1])
}
}
func (pool *proxyPool) Next() string{ // Function to give proxies in round robin format
if pool.count < len(*pool.proxy){
data := (*pool.proxy)[pool.count]
poolMutex.Lock()
pool.count = pool.count + 1
poolMutex.Unlock()
return prepProxy(data)
} else{
poolMutex.Lock()
pool.count = 0
poolMutex.Unlock()
return prepProxy((*pool.proxy)[pool.count])
}
}
func readFileByLine(filename string) (error,[]string){
data := []string{}
file, err := os.Open(filename)
if err != nil{
return err,[]string{}
} else {
scanner := bufio.NewScanner(file)
for scanner.Scan() {
data = append(data, scanner.Text())
}
if err := scanner.Err(); err != nil {
return err,[]string{}
}
err := file.Close()
if err != nil {
return err,[]string{}
}
return nil,data
}
}
func main(){
err , proxies := readFileByLine("proxies.txt") // Loads Proxies from file to slice
if err != nil{
fmt.Println("Error While Getting Proxies", err)
os.Exit(1)
}
ProxyPool := proxyPool{ // Initialization of ProxyPool
proxy: &proxies,
count: 0,
}
for i:=0; i<10; i++{
fmt.Println("Proxy:", ProxyPool.Next()) // Access the proxy with ProxyPool.Next()
}
/*Access proxies in go routine i.e., Threading*/
var wg sync.WaitGroup
wg.Add(10)
for i:=0; i<10; i++{ // Creates 10 Go routines
go func(){
fmt.Println("Proxy:", ProxyPool.Next()) // Access the proxy with ProxyPool.Next()
wg.Done()
}()
}
wg.Wait()
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment