Created
May 25, 2023 00:30
-
-
Save fulldump/80f4a2229d0b45006880e6586feb5cff to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"crypto/tls" | |
"encoding/json" | |
"fmt" | |
"io" | |
"io/ioutil" | |
"net/http" | |
"net/url" | |
"os" | |
"sync" | |
"sync/atomic" | |
"time" | |
) | |
var fetch = NewFetchWordsRAE() | |
var letters = []string{"a", "á", "b", "c", "d", "e", "é", "f", "g", "h", "i", "í", "j", "k", "l", "m", "ñ", "o", "ó", "p", "q", "r", "s", "t", "u", "ú", "ü", "v", "w", "x", "y", "z", "-"} | |
var fullList = 10 | |
var tasks = make(chan string, 200000) | |
var workers = 10 | |
var requests int64 | |
func main() { | |
go func() { | |
time.Sleep(60 * time.Minute) | |
os.Exit(0) | |
}() | |
go func() { | |
for { | |
fmt.Fprintln(os.Stderr, "pending tasks:", len(tasks), "requests:", requests) | |
time.Sleep(1 * time.Second) | |
} | |
}() | |
wg := &sync.WaitGroup{} | |
for i := 0; i < workers; i++ { | |
wg.Add(1) | |
go func() { | |
defer wg.Done() | |
for prefix := range tasks { | |
words, err := fetch(prefix) | |
atomic.AddInt64(&requests, 1) | |
if err != nil { | |
fmt.Fprintln(os.Stderr, "ERROR", err) | |
time.Sleep(30 * time.Second) | |
tasks <- prefix | |
continue | |
} | |
if len(words) < fullList { | |
for _, word := range words { | |
fmt.Println(word) | |
} | |
continue | |
} | |
if inArray(prefix, words) { | |
fmt.Println(prefix) | |
} | |
for _, letter := range letters { | |
tasks <- prefix + letter | |
} | |
} | |
}() | |
} | |
for _, letter := range letters { | |
tasks <- letter | |
} | |
wg.Wait() | |
} | |
func dive(prefix string) { | |
for _, letter := range letters { | |
newPrefix := prefix + letter | |
words, _ := fetch(newPrefix) | |
if len(words) == fullList { | |
if inArray(newPrefix, words) { | |
fmt.Println(newPrefix) | |
} | |
dive(newPrefix) | |
continue | |
} | |
for _, word := range words { | |
fmt.Println(word) | |
} | |
} | |
} | |
func inArray(item string, items []string) bool { | |
for _, i := range items { | |
if i == item { | |
return true | |
} | |
} | |
return false | |
} | |
func NewFetchWordsRAE() func(prefix string) ([]string, error) { | |
httpClient := &http.Client{ | |
Transport: &http.Transport{ | |
TLSClientConfig: &tls.Config{ | |
CurvePreferences: []tls.CurveID{tls.CurveP256, tls.CurveP384, tls.CurveP521, tls.X25519}, | |
}, | |
}, | |
} | |
return func(prefix string) (result []string, err error) { | |
escapedTerm := url.QueryEscape(prefix) | |
req, err := http.NewRequest("GET", "https://dle.rae.es/srv/keys?q="+escapedTerm, nil) | |
if err != nil { | |
return nil, err | |
} | |
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0") | |
// req.Header.Set("User-Agent", browser.Firefox()) | |
resp, err := httpClient.Do(req) | |
if err != nil { | |
return nil, err | |
} | |
defer resp.Body.Close() | |
var allBody []byte | |
defer func() { | |
if err == nil { | |
return | |
} | |
fmt.Fprintln(os.Stderr, resp.Status) | |
fmt.Fprintln(os.Stderr, resp.Header) | |
fmt.Fprintln(os.Stderr, string(allBody)) | |
}() | |
allBody, err = ioutil.ReadAll(resp.Body) | |
if err != nil { | |
return nil, err | |
} | |
err = json.Unmarshal(allBody, &result) | |
if err != nil { | |
return nil, err | |
} | |
io.Copy(ioutil.Discard, resp.Body) // To consume possible trailing characters after the JSON ends | |
return result, nil | |
} | |
} | |
func fetchWords(prefix string) ([]string, error) { | |
escapedTerm := url.QueryEscape(prefix) | |
req, err := http.NewRequest("GET", "https://dle.rae.es/srv/keys?q="+escapedTerm, nil) | |
if err != nil { | |
return nil, err | |
} | |
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0") | |
httpClient := &http.Client{ | |
Transport: &http.Transport{ | |
TLSClientConfig: &tls.Config{ | |
CurvePreferences: []tls.CurveID{tls.CurveP256, tls.CurveP384, tls.CurveP521, tls.X25519}, | |
}, | |
}, | |
} | |
resp, err := httpClient.Do(req) | |
if err != nil { | |
return nil, err | |
} | |
defer resp.Body.Close() | |
result := []string{} | |
err = json.NewDecoder(resp.Body).Decode(&result) | |
if err != nil { | |
return nil, err | |
} | |
io.Copy(ioutil.Discard, resp.Body) // To consume possible trailing characters after the JSON ends | |
return result, nil | |
} | |
// https://dle.rae.es/srv/keys?q=al | |
var mockData = map[string][]string{ | |
"a": {"a", "a-", "aba", "abaá", "ababillarse", "ababol", "abacá", "abacal", "abacalera", "abacalero"}, | |
"ab": {"aba", "abaá", "ababillarse", "ababol", "abacá", "abacal", "abacalera", "abacalero", "abacera", "abacería"}, | |
"aba": {"aba", "abaá", "ababillarse", "ababol", "abacá", "abacal", "abacalera", "abacalero", "abacera", "abacería"}, | |
"abab": {"ababillarse", "ababol"}, | |
} | |
func fetchWordsMock(prefix string) ([]string, error) { | |
result, ok := mockData[prefix] | |
if ok { | |
return result, nil | |
} | |
return []string{}, nil | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment