Skip to content

Instantly share code, notes, and snippets.

@eliben
Created February 12, 2021 14:54
Show Gist options
  • Save eliben/bfa2787cf4c6185ae369e869ddf54197 to your computer and use it in GitHub Desktop.
Save eliben/bfa2787cf4c6185ae369e869ddf54197 to your computer and use it in GitHub Desktop.
// StackOverflow analysis using its API in Go.
//
// Eli Bendersky [https://eli.thegreenplace.net]
// This code is in the public domain.
package main
import (
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/url"
"path/filepath"
"strconv"
"strings"
"time"
)
// Base query built with the explorer on
// https://api.stackexchange.com/docs/questions
// :
// "https://api.stackexchange.com/2.2/questions?page=2&pagesize=100&fromdate=1610409600&todate=1613088000&order=desc&sort=activity&tagged=go&site=stackoverflow"
// Generated with https://mholt.github.io/json-to-go/
type Reply struct {
Items []struct {
Tags []string `json:"tags"`
Owner struct {
Reputation int `json:"reputation"`
UserID int `json:"user_id"`
UserType string `json:"user_type"`
ProfileImage string `json:"profile_image"`
DisplayName string `json:"display_name"`
Link string `json:"link"`
} `json:"owner"`
IsAnswered bool `json:"is_answered"`
ViewCount int `json:"view_count"`
AcceptedAnswerID int `json:"accepted_answer_id,omitempty"`
AnswerCount int `json:"answer_count"`
Score int `json:"score"`
LastActivityDate int `json:"last_activity_date"`
CreationDate int `json:"creation_date"`
LastEditDate int `json:"last_edit_date"`
QuestionID int `json:"question_id"`
ContentLicense string `json:"content_license"`
Link string `json:"link"`
Title string `json:"title"`
} `json:"items"`
HasMore bool `json:"has_more"`
QuotaMax int `json:"quota_max"`
QuotaRemaining int `json:"quota_remaining"`
Total int `json:"total"`
}
func makePageQuery(page int, fromDate time.Time, toDate time.Time) string {
v := url.Values{}
v.Set("page", strconv.Itoa(page))
v.Set("pagesize", strconv.Itoa(100))
v.Set("fromdate", strconv.FormatInt(fromDate.Unix(), 10))
v.Set("todate", strconv.FormatInt(toDate.Unix(), 10))
v.Set("order", "desc")
v.Set("sort", "activity")
v.Set("tagged", "go")
v.Set("site", "stackoverflow")
return v.Encode()
}
func fetchResultsToDir(dir string, fromDate time.Time, toDate time.Time) {
for page := 1; ; page++ {
qs := makePageQuery(page, fromDate, toDate)
url := "https://api.stackexchange.com/2.2/questions?" + qs
fmt.Println(url)
resp, err := http.Get(url)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
fmt.Println("Response status:", resp.Status)
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
pageFilename := fmt.Sprintf("%s/so%03d.json", dir, page)
err = ioutil.WriteFile(pageFilename, body, 0644)
if err != nil {
log.Fatal(err)
}
fmt.Println("Wrote", pageFilename)
var reply Reply
if err = json.Unmarshal(body, &reply); err != nil {
log.Fatal(err)
}
if !reply.HasMore {
break
}
}
}
func main() {
fetchFlag := flag.Bool("fetch", false, "fetch new results from StackOverflow API")
dirFlag := flag.String("dir", "", "directory to store results")
flag.Parse()
if *fetchFlag {
fetchResultsToDir(*dirFlag, time.Date(2021, 1, 12, 0, 0, 0, 0, time.UTC), time.Date(2021, 2, 12, 0, 0, 0, 0, time.UTC))
}
fileinfos, err := ioutil.ReadDir(*dirFlag)
if err != nil {
log.Fatal(err)
}
totalNum := 0
numNegative := 0
numNegativeUnanswered := 0
for _, entry := range fileinfos {
if strings.HasSuffix(entry.Name(), "json") {
data, err := ioutil.ReadFile(filepath.Join(*dirFlag, entry.Name()))
if err != nil {
log.Fatal(err)
}
var reply Reply
if err = json.Unmarshal(data, &reply); err != nil {
log.Fatal(err)
}
totalNum += len(reply.Items)
for _, item := range reply.Items {
if item.Score < 0 {
numNegative++
fmt.Println(item.Score, item.Link)
if item.AnswerCount == 0 {
numNegativeUnanswered++
}
}
}
}
}
fmt.Println("Total", totalNum)
fmt.Println("Negative", numNegative)
fmt.Println("Negative unanswered", numNegativeUnanswered)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment