crawl.go

// Simple Go code for crawling the Fediverse on a single thread!

package main

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

func filter(peers []string) []string {
	toSkip := []string{"activitypub-troll.cf"}
	var fPeers []string

	for _, p := range peers {
		remove := false
		for _, skip := range toSkip {
			if strings.Contains(p, skip) {
				remove = true
				break
			}
		}

		if !remove {
			fPeers = append(fPeers, p)
		}
	}

	return fPeers
}

func contains(arr []string, target string) bool {
	for _, p := range arr {
		if p == target {
			return true
		}
	}
	return false
}

func getPeers(instance string) []string {
	peers := []string{}
	url := "https://" + instance + "/api/v1/instance/peers"

	client := http.Client{
		Timeout: 1 * time.Second,
	}

	res, err := client.Get(url)
	if err != nil {
		return peers
		//log.Fatal(err)
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return peers
		//log.Fatal(err)
	}

	err = json.Unmarshal(body, &peers)
	if err != nil {
		return peers
		//log.Fatal(err)
	}

	return filter(peers)
}

func main() {
	file, err := os.Create("peer_net.csv")
	writer := csv.NewWriter(file)

	defer writer.Flush()
	defer file.Close()

	if err != nil {
		log.Fatalf("failed creating file: %s", err)
	}

	row := []string{"source", "target"}
	writer.Write(row)

	var queue []string
	var processed []string

	seedInstance := "mastodon.social"
	queue = append(queue, seedInstance)

	prev := 0

	for len(queue) > 0 {
		instance := queue[0]
		queue = queue[1:]

		peers := getPeers(instance)

		for _, p := range peers {
			row = []string{instance, p}
			writer.Write(row)

			if !contains(queue, p) && !contains(processed, p) {
				queue = append(queue, p)
			}
		}

		processed = append(processed, instance)

		fmt.Print("In queue ", len(queue), " Processed ", len(processed))

		diff := len(queue) - prev
		if diff > 0 {
			fmt.Print(" Added ", diff)
		}

		fmt.Println()
		prev = len(queue)
	}
}