raedl.go

package main

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"sync"
	"sync/atomic"
	"time"
)

var fetch = NewFetchWordsRAE()
var letters = []string{"a", "á", "b", "c", "d", "e", "é", "f", "g", "h", "i", "í", "j", "k", "l", "m", "ñ", "o", "ó", "p", "q", "r", "s", "t", "u", "ú", "ü", "v", "w", "x", "y", "z", "-"}
var fullList = 10
var tasks = make(chan string, 200000)
var workers = 10
var requests int64

func main() {

	go func() {
		time.Sleep(60 * time.Minute)
		os.Exit(0)
	}()

	go func() {
		for {
			fmt.Fprintln(os.Stderr, "pending tasks:", len(tasks), "requests:", requests)
			time.Sleep(1 * time.Second)
		}
	}()

	wg := &sync.WaitGroup{}
	for i := 0; i < workers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()

			for prefix := range tasks {

				words, err := fetch(prefix)
				atomic.AddInt64(&requests, 1)
				if err != nil {
					fmt.Fprintln(os.Stderr, "ERROR", err)
					time.Sleep(30 * time.Second)
					tasks <- prefix
					continue
				}
				if len(words) < fullList {
					for _, word := range words {
						fmt.Println(word)
					}
					continue
				}
				if inArray(prefix, words) {
					fmt.Println(prefix)
				}

				for _, letter := range letters {
					tasks <- prefix + letter
				}
			}

		}()
	}

	for _, letter := range letters {
		tasks <- letter
	}

	wg.Wait()
}

func dive(prefix string) {
	for _, letter := range letters {
		newPrefix := prefix + letter
		words, _ := fetch(newPrefix)
		if len(words) == fullList {
			if inArray(newPrefix, words) {
				fmt.Println(newPrefix)
			}
			dive(newPrefix)
			continue
		}
		for _, word := range words {
			fmt.Println(word)
		}
	}
}

func inArray(item string, items []string) bool {
	for _, i := range items {
		if i == item {
			return true
		}
	}

	return false
}

func NewFetchWordsRAE() func(prefix string) ([]string, error) {

	httpClient := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				CurvePreferences: []tls.CurveID{tls.CurveP256, tls.CurveP384, tls.CurveP521, tls.X25519},
			},
		},
	}

	return func(prefix string) (result []string, err error) {

		escapedTerm := url.QueryEscape(prefix)

		req, err := http.NewRequest("GET", "https://dle.rae.es/srv/keys?q="+escapedTerm, nil)
		if err != nil {
			return nil, err
		}
		req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0")
		// req.Header.Set("User-Agent", browser.Firefox())

		resp, err := httpClient.Do(req)
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()

		var allBody []byte

		defer func() {
			if err == nil {
				return
			}
			fmt.Fprintln(os.Stderr, resp.Status)
			fmt.Fprintln(os.Stderr, resp.Header)
			fmt.Fprintln(os.Stderr, string(allBody))
		}()

		allBody, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}

		err = json.Unmarshal(allBody, &result)
		if err != nil {
			return nil, err
		}
		io.Copy(ioutil.Discard, resp.Body) // To consume possible trailing characters after the JSON ends

		return result, nil

	}
}

func fetchWords(prefix string) ([]string, error) {

	escapedTerm := url.QueryEscape(prefix)

	req, err := http.NewRequest("GET", "https://dle.rae.es/srv/keys?q="+escapedTerm, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0")

	httpClient := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				CurvePreferences: []tls.CurveID{tls.CurveP256, tls.CurveP384, tls.CurveP521, tls.X25519},
			},
		},
	}

	resp, err := httpClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	result := []string{}

	err = json.NewDecoder(resp.Body).Decode(&result)
	if err != nil {
		return nil, err
	}
	io.Copy(ioutil.Discard, resp.Body) // To consume possible trailing characters after the JSON ends

	return result, nil
}

// https://dle.rae.es/srv/keys?q=al
var mockData = map[string][]string{
	"a":    {"a", "a-", "aba", "abaá", "ababillarse", "ababol", "abacá", "abacal", "abacalera", "abacalero"},
	"ab":   {"aba", "abaá", "ababillarse", "ababol", "abacá", "abacal", "abacalera", "abacalero", "abacera", "abacería"},
	"aba":  {"aba", "abaá", "ababillarse", "ababol", "abacá", "abacal", "abacalera", "abacalero", "abacera", "abacería"},
	"abab": {"ababillarse", "ababol"},
}

func fetchWordsMock(prefix string) ([]string, error) {
	result, ok := mockData[prefix]
	if ok {
		return result, nil
	}

	return []string{}, nil
}