Feeds.go

package main

import (
	"github.com/gocql/gocql"
	"github.com/SlyMarbo/rss"
	"log"
	"crypto/md5"
	"encoding/hex"
	"time"
	"fmt"
)

func main(){
	cluster := gocql.NewCluster("localhost")
	cluster.Keyspace = "hashmedia"
	cluster.Consistency = gocql.Quorum
	session, _ := cluster.CreateSession()
	defer session.Close()
	getGetLinks(session)
}

func getGetLinks(session *gocql.Session) {
	feeds := session.Query("SELECT zone,feedLink,sitecode FROM feeds").Iter()
	fmt.Println("Reading Feeds from the Database Start Time: ", time.Now())
	var feedLink string
	var zone string
	var siteCode string
	defer feeds.Close()
	for feeds.Scan(&zone, &feedLink, &siteCode) {
		fetchLinks(session,feedLink,zone,siteCode)
	}
	if err := feeds.Close(); err != nil {
		log.Fatal("The Feeds Read Through Error",err)
	}
	fmt.Println("Reading Feeds from the Database End Time: ", time.Now())
}

func GetMD5Hash(text string) string {
	hasher := md5.New()
	hasher.Write([]byte(text))
	return hex.EncodeToString(hasher.Sum(nil))
}

func fetchLinks(session *gocql.Session,feedLink string,zone string,siteCode string){
	feed, err := rss.Fetch(feedLink)
	if err == nil{
		var links = feed.Items
		for _, link := range links {
			if err := session.Query(`INSERT INTO links  (zone,linkhash,datepublished,site,url,sitecode) VALUES (?,?,?,?,?,?)`,
				zone, GetMD5Hash(link.ID), link.Date, link.Link, link.ID, siteCode).Exec(); err != nil {
				log.Fatal(err)
			}
		}
	}
}