Skip to content

Instantly share code, notes, and snippets.

@diego-augusto
Created April 6, 2024 00:54
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save diego-augusto/34245e3fcefb37dbc7c2fb1caf1103b7 to your computer and use it in GitHub Desktop.
Save diego-augusto/34245e3fcefb37dbc7c2fb1caf1103b7 to your computer and use it in GitHub Desktop.
package main
import (
"context"
"log"
"os"
"os/signal"
"sync"
"time"
)
type Proxies struct {
Proxies []string
current int
m sync.Mutex
}
func NewProxies(proxies ...string) *Proxies {
return &Proxies{
current: 0,
Proxies: proxies,
}
}
func (proxies *Proxies) Len() int {
proxies.m.Lock()
defer proxies.m.Unlock()
return len(proxies.Proxies)
}
func (proxies *Proxies) GetNext() string {
proxies.m.Lock()
defer proxies.m.Unlock()
proxy := proxies.Proxies[proxies.current]
proxies.current = (proxies.current + 1) % len(proxies.Proxies)
return proxy
}
type Job struct {
ID int
URL []string
Proxy string
}
func (j Job) Do(ctx context.Context) {
for _, url := range j.URL {
select {
case <-ctx.Done():
log.Println("Job ID:", j.ID, "URL:", url, "Proxy:", j.Proxy, "Canceled")
return
case <-time.After(1 * time.Second):
log.Println("Job ID:", j.ID, "URL:", url, "Proxy:", j.Proxy)
}
}
}
type Executer interface {
Do(ctx context.Context)
}
type Worker struct {
Jobs []Executer
}
func (w Worker) Run(ctx context.Context, workers int) {
var wg sync.WaitGroup
jobChannel := make(chan Executer, len(w.Jobs))
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for job := range jobChannel {
job.Do(ctx)
}
}()
}
for _, job := range w.Jobs {
jobChannel <- job
}
close(jobChannel)
wg.Wait()
}
func main() {
proxies := NewProxies("proxy1", "proxy2")
jobs := []Job{
{ID: 1, URL: []string{"http://example.com", "http://foo.com", "http://bar.com"}, Proxy: proxies.GetNext()},
{ID: 2, URL: []string{"http://example.net"}, Proxy: proxies.GetNext()},
{ID: 3, URL: []string{"http://example.io"}, Proxy: proxies.GetNext()},
{ID: 4, URL: []string{"http://example.co"}, Proxy: proxies.GetNext()},
{ID: 5, URL: []string{"http://example.info"}, Proxy: proxies.GetNext()},
{ID: 6, URL: []string{"http://example.biz"}, Proxy: proxies.GetNext()},
{ID: 7, URL: []string{"http://example.us"}, Proxy: proxies.GetNext()},
{ID: 8, URL: []string{"http://example.tv"}, Proxy: proxies.GetNext()},
{ID: 9, URL: []string{"http://example.name"}, Proxy: proxies.GetNext()},
{ID: 10, URL: []string{"http://example.mobi"}, Proxy: proxies.GetNext()},
}
executers := make([]Executer, len(jobs))
for i, job := range jobs {
executers[i] = job
}
worker := Worker{Jobs: executers}
exit := make(chan os.Signal, 1)
signal.Notify(exit, os.Interrupt)
ctx, cancel := context.WithCancel(context.Background())
go func() {
<-exit
cancel()
}()
for {
select {
case <-ctx.Done():
cancel()
log.Println("Shutting down...")
os.Exit(0)
default:
worker.Run(ctx, proxies.Len())
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment