dev_to_emails.go

dev_to_emails.go

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"strings"
	"time"
)

func main() {
	devToCookiePtr := flag.String("devtocookie", "", "")
	githubTokenPtr := flag.String("ghtoken", "", "")
	flag.Parse()

	if devToCookiePtr == nil || *devToCookiePtr == "" || githubTokenPtr == nil || *githubTokenPtr == "" {
		fmt.Println("usage: go run dev_to_emails.go --devtocookie=cookie --ghtoken=token")
		return
	}

	users, err := getDevToUsers(0, *devToCookiePtr, []devToUser{})
	fmt.Printf("fetched %d dev.to users\n", len(users))
	if err != nil {
		fmt.Println(err)
		return
	}

	// persist users in users.json
	file, err := json.MarshalIndent(users, "", " ")
	if err != nil {
		fmt.Println(err)
		return
	}

	if err = os.WriteFile("users.json", file, 0644); err != nil {
		fmt.Println(err)
		return
	}

	findGithubEmails(users, *githubTokenPtr)
}

// get users from dev.to
func getDevToUsers(page int, cookie string, users []devToUser) ([]devToUser, error) {
	fmt.Printf("fetching page %d...\n", page)

	httpClient := http.Client{
		Timeout: time.Second * 5,
	}
	req, err := http.NewRequest("GET", fmt.Sprintf("https://dev.to/api/followers/users?page=%d", page), nil)
	if err != nil {
		return users, err
	}

	req.Header.Set("Cookie", fmt.Sprintf("_Devto_Forem_Session=%s", cookie))
	res, err := httpClient.Do(req)
	if err != nil {
		return users, err
	}

	defer res.Body.Close()
	pageUsers := []devToUser{}
	if err = json.NewDecoder(res.Body).Decode(&pageUsers); err != nil {
		return users, err
	}

	users = append(users, pageUsers...)
	if len(pageUsers) == 0 {
		return users, nil
	}

	return getDevToUsers(page+1, cookie, users)
}

func findGithubEmails(users []devToUser, ghToken string) {
	// find github emails
	uniqueEmails := make(map[string]bool)
	for _, user := range users {
		email, err := getGithubUserEmail(ghToken, user.Username, false)
		if err != nil {
			fmt.Println(err)
			return
		}

		if email == "" || strings.Contains(email, "noreply") || strings.Contains(email, "github.com") {
			continue
		}

		fmt.Printf("github email for %s: %s\n", user.Username, email)
		uniqueEmails[email] = true
	}

	fmt.Printf("found %d unique emails\n", len(uniqueEmails))

	// save emails into emails.txt, one email per line
	outFile, err := os.Create("emails.txt")
	if err != nil {
		fmt.Println(err)
		return
	}

	for email := range uniqueEmails {
		if _, err = outFile.WriteString(fmt.Sprintf("%s\n", email)); err != nil {
			fmt.Println(err)
			return
		}
	}
}

func getGithubUserEmail(token string, username string, wait bool) (string, error) {
	if wait {
		time.Sleep(time.Second * 1)
	}

	httpClient := http.Client{
		Timeout: time.Second * 5,
	}
	req, err := http.NewRequest("GET", fmt.Sprintf("https://api.github.com/users/%s/events", username), nil)
	if err != nil {
		return "", err
	}

	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
	res, err := httpClient.Do(req)
	if err != nil {
		return "", err
	}

	defer res.Body.Close()
	respBody, _ := io.ReadAll(res.Body)

	errMessage := githubError{}
	if err = json.Unmarshal(respBody, &errMessage); err == nil {
		if errMessage.Message != "" {
			if strings.Contains(errMessage.Message, "API rate limit") {
				// make another retry with some wait time
				return getGithubUserEmail(token, username, true)
			}

			return "", nil
		}
	}

	events := []githubEvent{}
	if err = json.Unmarshal(respBody, &events); err != nil {
		return "", err
	}

	for _, event := range events {
		for _, commit := range event.Payload.Commits {
			if commit.Author.Email != "" {
				return commit.Author.Email, nil
			}
		}
	}

	return "", nil
}

type devToUser struct {
	Username string `json:"username"`
}

type githubError struct {
	Message string `json:"message"`
}

type githubEvent struct {
	Payload githubEventPayload `json:"payload"`
}

type githubEventPayload struct {
	Commits []githubEventCommit `json:"commits"`
}

type githubEventCommit struct {
	Author githubEventAuthor `json:"author"`
}

type githubEventAuthor struct {
	Email string `json:"email"`
}