Skip to content

Instantly share code, notes, and snippets.

@aarondl
Created September 13, 2013 19:09
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save aarondl/6554786 to your computer and use it in GitHub Desktop.
Save aarondl/6554786 to your computer and use it in GitHub Desktop.
Autofan??
<!DOCTYPE html>
<html>
<head>
<link href="standard.css" rel="stylesheet" type="text/css">
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
<title>autofan pre-alpha</title>
<script type="text/javascript">
$(function(){
$("#submitbtn").click(function(){
$("#content").text("Loading...");
$("#content").load("/", { id: $('#idtext').val() });
});
});
</script>
</head>
<body>
<div id="body">
<h1>Autofan</h1>
<h2>Enter a fan club topic id:</h2>
<p>e.g. 196953</p>
<input id="idtext" type="text" /><br />
<input value="Find" type="button" id="submitbtn" class="peter-river-flat-button"/><br />
<div id="content">
Empty.
</div>
</div>
</body>
</html>
package main
import (
"strconv"
"io/ioutil"
"fmt"
"bytes"
"flag"
"net/http"
"regexp"
"log"
"os"
)
var (
indexbytes []byte
httpBind = flag.String("http", ":8888", "The bind address for the http server.")
test = flag.Int("test", 0, "A TL fan club id to test the parser with.")
)
const (
PageURI = "http://www.teamliquid.net/forum/viewmessage.php?topic_id=%v&currentpage=%v"
)
var (
rgxPost = regexp.MustCompile(`(?i)<td class='forumPost' width='100%'>(.*?)</td>`)
rgxSignup = regexp.MustCompile(`(?i)sign me(?: up)?|i'?ll join|add me|count me in`)
rgxDataUser = regexp.MustCompile(`data-user="(.*?)"`)
rgxNext = regexp.MustCompile(`<a href="\/forum\/viewmessage\.php\?topic_id=[0-9]+&amp;currentpage=[0-9]+" title="Next Page \([0-9]+\)" rel="next">Next</a>`)
)
type PostPage struct {
Posts int
Signups []string
}
func Parse(id int) ([]*PostPage, error) {
page := 1
hasNextPage := true
postPages := make([]*PostPage, 0)
for hasNextPage {
uri := fmt.Sprintf(PageURI, id, page)
log.Println("Making request: ", uri)
resp, err := http.Get(uri)
if err != nil {
return nil, err
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("Error retrieving uri (%v): %v", resp.StatusCode, uri)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
hasNextPage = rgxNext.Match(body)
postPage := new(PostPage)
rawPosts := rgxPost.FindAllSubmatch(body, -1)
users := rgxDataUser.FindAllSubmatch(body, -1)
postPage.Posts = len(rawPosts)
postPage.Signups = make([]string, 0)
for i, rawPost := range rawPosts {
if rgxSignup.Match(rawPost[1]) {
postPage.Signups = append(postPage.Signups, string(users[i][1]))
log.Printf("Signup found: %s", users[i][1])
}
}
postPages = append(postPages, postPage)
page++
}
return postPages, nil
}
func handleIndex(w http.ResponseWriter, r *http.Request) {
log.Println("Request from:", r.RemoteAddr, r.RequestURI)
if r.URL.Path != "/" {
return
}
if sid := r.FormValue("id"); len(sid) > 0 {
id, err := strconv.Atoi(sid)
if err != nil {
fmt.Println("Bad id:", sid, err)
w.WriteHeader(500)
return
}
pages, err := Parse(id)
if err != nil {
fmt.Println("Error parsing:", err)
w.WriteHeader(500)
}
var buf bytes.Buffer
for _, page := range pages {
for _, signup := range page.Signups {
_, err = buf.WriteString(signup)
if err != nil {
log.Fatal(err)
}
_, err = buf.WriteString("<br />")
if err != nil {
log.Fatal(err)
}
}
}
_, err = buf.WriteTo(w)
if err != nil {
fmt.Println("Error writing response:", err)
}
return
}
_, err := w.Write(indexbytes)
if err != nil {
log.Println("Error writing response:", err)
}
}
func main() {
var err error
var indexhtml, standardcss *os.File
flag.Parse()
if *test > 0 {
var pages []*PostPage
pages, err = Parse(*test)
if err != nil {
log.Fatal(err)
}
log.Println("Found:", len(pages), "pages.")
var buf bytes.Buffer
for _, page := range pages {
buf.Reset()
for _, signup := range page.Signups {
_, err = buf.WriteString(signup)
if err != nil {
log.Fatal(err)
}
_, err = buf.WriteString(", ")
if err != nil {
log.Fatal(err)
}
}
log.Printf("Page %v (%v): %v", len(pages), len(page.Signups), buf.String())
}
log.Println("Exiting.")
return
}
indexhtml, err = os.Open("index.html")
defer indexhtml.Close()
indexbytes, err = ioutil.ReadAll(indexhtml)
if err != nil {
log.Fatal(err)
}
standardcss, err = os.Open("standard.css")
if err != nil {
log.Fatal(err)
}
defer standardcss.Close()
http.HandleFunc("/", handleIndex)
http.Handle("/standard.css", http.FileServer(http.Dir(".")))
err = http.ListenAndServe(*httpBind, nil)
if err != nil {
log.Fatal(err)
}
log.Println("Exiting.")
}
* {font-family: Sans-Serif;}
body {background-color #f9f9f9;}
#body {margin: 0 auto; width: 700px;}
.peter-river-flat-button
{
width: 220px;
height: 60px;
font-size: 22px;
color: white;
text-align: center;
text-shadow: 0px 1px 2px rgba(0,0,0,.25);
background: #3498db;
border: 0;
border-bottom: 2px solid #2a8bcc;
cursor: pointer;
-webkit-box-shadow: inset 0 -2px #2a8bcc;
box-shadow: inset 0 -2px ##2a8bcc;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment