fetch-all-questions.go

// StackOverflow analysis using its API in Go.
//
// Eli Bendersky [https://eli.thegreenplace.net]
// This code is in the public domain.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

// Base query built with the explorer on
// https://api.stackexchange.com/docs/questions
// :
// "https://api.stackexchange.com/2.2/questions?page=2&pagesize=100&fromdate=1610409600&todate=1613088000&order=desc&sort=activity&tagged=go&site=stackoverflow"

// Generated with https://mholt.github.io/json-to-go/
type Reply struct {
	Items []struct {
		Tags  []string `json:"tags"`
		Owner struct {
			Reputation   int    `json:"reputation"`
			UserID       int    `json:"user_id"`
			UserType     string `json:"user_type"`
			ProfileImage string `json:"profile_image"`
			DisplayName  string `json:"display_name"`
			Link         string `json:"link"`
		} `json:"owner"`
		IsAnswered       bool   `json:"is_answered"`
		ViewCount        int    `json:"view_count"`
		AcceptedAnswerID int    `json:"accepted_answer_id,omitempty"`
		AnswerCount      int    `json:"answer_count"`
		Score            int    `json:"score"`
		LastActivityDate int    `json:"last_activity_date"`
		CreationDate     int    `json:"creation_date"`
		LastEditDate     int    `json:"last_edit_date"`
		QuestionID       int    `json:"question_id"`
		ContentLicense   string `json:"content_license"`
		Link             string `json:"link"`
		Title            string `json:"title"`
	} `json:"items"`
	HasMore        bool `json:"has_more"`
	QuotaMax       int  `json:"quota_max"`
	QuotaRemaining int  `json:"quota_remaining"`
	Total          int  `json:"total"`
}

func makePageQuery(page int, fromDate time.Time, toDate time.Time) string {
	v := url.Values{}
	v.Set("page", strconv.Itoa(page))
	v.Set("pagesize", strconv.Itoa(100))
	v.Set("fromdate", strconv.FormatInt(fromDate.Unix(), 10))
	v.Set("todate", strconv.FormatInt(toDate.Unix(), 10))
	v.Set("order", "desc")
	v.Set("sort", "activity")
	v.Set("tagged", "go")
	v.Set("site", "stackoverflow")
	return v.Encode()
}

func fetchResultsToDir(dir string, fromDate time.Time, toDate time.Time) {
	for page := 1; ; page++ {
		qs := makePageQuery(page, fromDate, toDate)
		url := "https://api.stackexchange.com/2.2/questions?" + qs
		fmt.Println(url)

		resp, err := http.Get(url)
		if err != nil {
			log.Fatal(err)
		}
		defer resp.Body.Close()

		fmt.Println("Response status:", resp.Status)
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Fatal(err)
		}

		pageFilename := fmt.Sprintf("%s/so%03d.json", dir, page)
		err = ioutil.WriteFile(pageFilename, body, 0644)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Println("Wrote", pageFilename)

		var reply Reply
		if err = json.Unmarshal(body, &reply); err != nil {
			log.Fatal(err)
		}

		if !reply.HasMore {
			break
		}
	}
}

func main() {
	fetchFlag := flag.Bool("fetch", false, "fetch new results from StackOverflow API")
	dirFlag := flag.String("dir", "", "directory to store results")

	flag.Parse()

	if *fetchFlag {
		fetchResultsToDir(*dirFlag, time.Date(2021, 1, 12, 0, 0, 0, 0, time.UTC), time.Date(2021, 2, 12, 0, 0, 0, 0, time.UTC))
	}

	fileinfos, err := ioutil.ReadDir(*dirFlag)
	if err != nil {
		log.Fatal(err)
	}

	totalNum := 0
	numNegative := 0
	numNegativeUnanswered := 0

	for _, entry := range fileinfos {
		if strings.HasSuffix(entry.Name(), "json") {
			data, err := ioutil.ReadFile(filepath.Join(*dirFlag, entry.Name()))
			if err != nil {
				log.Fatal(err)
			}

			var reply Reply
			if err = json.Unmarshal(data, &reply); err != nil {
				log.Fatal(err)
			}

			totalNum += len(reply.Items)

			for _, item := range reply.Items {
				if item.Score < 0 {
					numNegative++
					fmt.Println(item.Score, item.Link)

					if item.AnswerCount == 0 {
						numNegativeUnanswered++
					}
				}
			}
		}
	}

	fmt.Println("Total", totalNum)
	fmt.Println("Negative", numNegative)
	fmt.Println("Negative unanswered", numNegativeUnanswered)
}