Markov chain generator based on Breitbart comments

app.go

package markov

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"strings"

	"cloud.google.com/go/storage"
	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/file"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/urlfetch"
)

const (
	apiKey    = "TQGVKnEtvReZD2L0c4f1kpM7zrsbGeQwtFz9jtBorO6c1BRyXMLdHvs29yRhJBQC"
	numPages  = 20
	prefixLen = 2
	object    = "chain.json"
)

func init() {
	http.HandleFunc("/cron", cronHandler)
}

func cronHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)

	var buf bytes.Buffer
	var cursor string
	for i := 0; i < numPages; i++ {
		resp, err := getPage(ctx, cursor)
		if err != nil {
			log.Errorf(ctx, "getPage: %v", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		cursor = resp.Cursor.Next
		for _, msg := range resp.Response {
			buf.WriteString(msg.RawMessage)
		}
	}

	c := NewChain(prefixLen)
	c.Build(&buf)

	var buf2 bytes.Buffer
	if err := json.NewEncoder(&buf2).Encode(c.chain); err != nil {
		log.Errorf(ctx, "json.Encode: %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	bucket, err := file.DefaultBucketName(ctx)
	if err != nil {
		log.Errorf(ctx, "DefaultBucketName: %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	client, err := storage.NewClient(ctx)
	if err != nil {
		log.Errorf(ctx, "storage.NewClient: %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	obj := client.Bucket(bucket).Object(object)
	ow := obj.NewWriter(ctx)
	if _, err := io.Copy(ow, &buf2); err != nil {
		log.Errorf(ctx, "io.Copy: %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if err := ow.Close(); err != nil {
		log.Errorf(ctx, "obj.Close: %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if err := obj.ACL().Set(ctx, storage.AllUsers, storage.RoleReader); err != nil {
		log.Errorf(ctx, "obj.Acl.Set: %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	fmt.Fprintln(w, "done")
}

type response struct {
	Cursor struct {
		Next string `json:"next"`
	} `json:"cursor"`
	Response []struct {
		RawMessage string `json:"raw_message"`
	} `json:"response"`
}

func getPage(ctx context.Context, c string) (*response, error) {
	url := "https://disqus.com/api/3.0/forums/listPosts.json?forum=breitbartproduction&limit=100&api_key=" + apiKey
	if c != "" {
		url += "&cursor=" + c
	}
	resp, err := urlfetch.Client(ctx).Get(url)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status)
	}
	defer resp.Body.Close()
	var r response
	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
		return nil, err
	}
	return &r, nil
}

// Prefix is a Markov chain prefix of one or more words.
type Prefix []string

// String returns the Prefix as a string (for use as a map key).
func (p Prefix) String() string {
	return strings.Join(p, " ")
}

// Shift removes the first word from the Prefix and appends the given word.
func (p Prefix) Shift(word string) {
	copy(p, p[1:])
	p[len(p)-1] = word
}

// Chain contains a map ("chain") of prefixes to a list of suffixes.
// A prefix is a string of prefixLen words joined with spaces.
// A suffix is a single word. A prefix can have multiple suffixes.
type Chain struct {
	chain     map[string][]string
	prefixLen int
}

// NewChain returns a new Chain with prefixes of prefixLen words.
func NewChain(prefixLen int) *Chain {
	return &Chain{make(map[string][]string), prefixLen}
}

// Build reads text from the provided Reader and
// parses it into prefixes and suffixes that are stored in Chain.
func (c *Chain) Build(r io.Reader) {
	br := bufio.NewReader(r)
	p := make(Prefix, c.prefixLen)
	for {
		var s string
		if _, err := fmt.Fscan(br, &s); err != nil {
			break
		}
		key := p.String()
		c.chain[key] = append(c.chain[key], s)
		p.Shift(s)
	}
}

// Generate returns a string of at most n length generated from Chain.
func (c *Chain) Generate(n int) string {
	p := make(Prefix, c.prefixLen)
	var words []string
	for i := 0; i < n; {
		choices := c.chain[p.String()]
		if len(choices) == 0 {
			break
		}
		next := choices[rand.Intn(len(choices))]
		words = append(words, next)
		p.Shift(next)
		i += len(next)
	}
	return strings.Join(words, " ")
}

app.yaml

runtime: go
api_version: go1

handlers:
- url: /
  static_files: index.html
  upload: index\.html

- url: /.*
  script: _go_app
  login: admin

cors.json

[
  {
    "origin": [
      "*",
      "http://breit-bot.appspot.com",
      "https://breit-bot.appspot.com"
    ],
    "method": ["GET"],
    "maxAgeSeconds": 3600
  }
]

cron.yaml

cron:
- description: "Hourly update"
  url: /cron
  schedule: every 1 hours

example

Too many. Sucker the Sucker must go Bwahahaha. Hi Hillary. How is it in the woods...you double loser. I think your crystal ball is defective... So once again, Trump has been right all along! Kudos to him for not letting CNN off the hook at press briefings. so sad..... Hey thousands are dying from opiods Make conceal and carry legal in Manhattan and California. Watch liberal heads explode. I had a problem with Milo already when he was at WH press briefings, he must have thought he was going to earn them points or fame but because they were the only ones who could. It's also stated when they arrive that any professor can award or take away points for the actions of a student. No rule change, just Dumbledore acknowledging hard work and people doing the right thing, even Neville was awarded points for trying to keep Harry and crew from going out that night. Fake voters are found in the cemetery . They are just trying to salvage the Democrat party. Look, we all know Russia does all it can to influence the political persuasion of the United States. The truth of the matter is that this is

index.html

<html>
<head>
<style>
body {
  font-family: "Helvetica Neue",arial,sans-serif;
}
</style>
</head>
<body>
	<h3>ALL HAIL BREITBOT</h3>

	<p><a href="#">Refresh</a></p>
	
	<span id="out"></span>

	<p><b>Disclaimer: The text on this site is generated by a computer,
		trained on comments from breitbart dot com. Nothing it says is
		anything like the opinion of the author of the code that
		generated it.</b></p>

	<p>Markov model baked fresh every hour.</p>
</body>
<script type="text/javascript">
const prefixLength = 2;
const num = 500;
const url = 'https://storage.googleapis.com/breit-bot.appspot.com/chain.json';

function get(cb) {
  var xhr = new XMLHttpRequest();
  xhr.open("GET", url, true);
  xhr.onreadystatechange = function() {
    if (xhr.readyState != 4) { return; }
    if (xhr.status >= 400) { console.log(xhr.responseText); }
    cb(JSON.parse(xhr.responseText));
  };
  xhr.send();
}

var data = {};
get(function(newData) {
  data = newData;
  refresh();
});

document.getElementsByTagName('a')[0].onclick = refresh;

function refresh() {
  var out = [];

  // Start with a random element.
  var keys = Object.keys(data);
  var rand = keys[Math.floor(keys.length * Math.random())];
  var prev = rand.split(' ');

  for (var i = 0; i < num; i++) {
    var choices = data[prev.join(' ')];
    if (choices == undefined) { return; }
    var next = choices[Math.floor(Math.random() * choices.length)];
    out.push(next);
    prev.push(next);
    if (prev.length > prefixLength) {
      prev.shift();
    }
  }
  document.getElementById('out').innerText = out.join(' ');

}
</script>
</html>