ingest.go

package main

import (
	"bufio"
	"github.com/gocql/gocql"
	"log"
	"os"
	"regexp"
)

type Movie struct {
	title string
	year  string
}

func main() {
	lines := make(chan string)
	movies := make(chan Movie)

	// Spin up a goroutine to read in movies
	go func() {
		file, err := os.Open("movies.list")
		defer file.Close()

		if err != nil {
			log.Fatal(err)
		}

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			lines <- scanner.Text()
		}

		if err := scanner.Err(); err != nil {
			log.Fatal(err)
		}

		close(lines)
	}()

	// Spin up a goroutine to parse the movie information
	go func() {
		// Compile a regex to parse the movie
		regex := regexp.MustCompile("(.*) \\((.*)\\)")

		for line := range lines {
			matches := regex.FindStringSubmatch(line)

			if len(matches) == 3 {
				movie := Movie{matches[1], matches[2]}
				movies <- movie
			} else {
				log.Printf("Could not parse line: " + line)
			}
		}

		close(movies)
	}()

	// Connect to the cluster
	cluster := gocql.NewCluster("127.0.0.1", "127.0.0.2", "127.0.0.3")
	cluster.Keyspace = "imdb"
	cluster.Consistency = gocql.Quorum
	session, err := cluster.CreateSession()
	defer session.Close()

	if err != nil {
		log.Fatal(err)
	}

	// Iterate over the movies and insert them into the cluster
	invalid_strings := 0
	for movie := range movies {
		if err := session.Query(`INSERT INTO movies (id, title, year) VALUES (now(), ?, ?)`, movie.title, movie.year).Exec(); err != nil {
			// log.Printf("%v:%v", title, err)
			invalid_strings = invalid_strings + 1
		}
	}
	log.Printf("Invalid strings: %v", invalid_strings)
}

schema.cql

CREATE KEYSPACE imdb WITH replication = {
  'class': 'SimpleStrategy',
  'replication_factor': '2'
};

USE imdb;

CREATE TABLE movies (
  id timeuuid,
  title text,
  year text,
  PRIMARY KEY ((id))
) WITH
  bloom_filter_fp_chance=0.010000 AND
  caching='KEYS_ONLY' AND
  comment='' AND
  dclocal_read_repair_chance=0.100000 AND
  gc_grace_seconds=864000 AND
  index_interval=128 AND
  read_repair_chance=0.000000 AND
  replicate_on_write='true' AND
  populate_io_cache_on_flush='false' AND
  default_time_to_live=0 AND
  speculative_retry='99.0PERCENTILE' AND
  memtable_flush_period_in_ms=0 AND
  compaction={'class': 'SizeTieredCompactionStrategy'} AND
  compression={'sstable_compression': 'LZ4Compressor'};