Created
June 6, 2021 05:13
-
-
Save White2001Offl/443080b0afba48bcb8508efca9ab41de to your computer and use it in GitHub Desktop.
ProxyPool in go with thread safe
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/*Its a simple implementation where follows round robin format to access proxies in thread or normal method*/ | |
/*The proxies should be loaded in format ip:port:user:pass*/ | |
import ( | |
"sync" | |
"fmt" | |
"os" | |
"bufio" | |
"strings" | |
) | |
var poolMutex = &sync.Mutex{} // Should be in global scope | |
type proxyPool struct{ // A Custom type for ProxyPool where stores the current count and proxies | |
proxy *[]string | |
count int | |
} | |
func prepProxy (proxy string) string{ // Returns the proxy which is compatible in GO Url Parse | |
splitter := strings.Split(proxy,":") | |
if len(splitter) == 2{ | |
return fmt.Sprintf("http://%v:%v",splitter[0],splitter[1]) | |
} else{ | |
return fmt.Sprintf("http://%v:%v@%v:%v",splitter[2],splitter[3],splitter[0],splitter[1]) | |
} | |
} | |
func (pool *proxyPool) Next() string{ // Function to give proxies in round robin format | |
if pool.count < len(*pool.proxy){ | |
data := (*pool.proxy)[pool.count] | |
poolMutex.Lock() | |
pool.count = pool.count + 1 | |
poolMutex.Unlock() | |
return prepProxy(data) | |
} else{ | |
poolMutex.Lock() | |
pool.count = 0 | |
poolMutex.Unlock() | |
return prepProxy((*pool.proxy)[pool.count]) | |
} | |
} | |
func readFileByLine(filename string) (error,[]string){ | |
data := []string{} | |
file, err := os.Open(filename) | |
if err != nil{ | |
return err,[]string{} | |
} else { | |
scanner := bufio.NewScanner(file) | |
for scanner.Scan() { | |
data = append(data, scanner.Text()) | |
} | |
if err := scanner.Err(); err != nil { | |
return err,[]string{} | |
} | |
err := file.Close() | |
if err != nil { | |
return err,[]string{} | |
} | |
return nil,data | |
} | |
} | |
func main(){ | |
err , proxies := readFileByLine("proxies.txt") // Loads Proxies from file to slice | |
if err != nil{ | |
fmt.Println("Error While Getting Proxies", err) | |
os.Exit(1) | |
} | |
ProxyPool := proxyPool{ // Initialization of ProxyPool | |
proxy: &proxies, | |
count: 0, | |
} | |
for i:=0; i<10; i++{ | |
fmt.Println("Proxy:", ProxyPool.Next()) // Access the proxy with ProxyPool.Next() | |
} | |
/*Access proxies in go routine i.e., Threading*/ | |
var wg sync.WaitGroup | |
wg.Add(10) | |
for i:=0; i<10; i++{ // Creates 10 Go routines | |
go func(){ | |
fmt.Println("Proxy:", ProxyPool.Next()) // Access the proxy with ProxyPool.Next() | |
wg.Done() | |
}() | |
} | |
wg.Wait() | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment