Code for comparing the ratings of books with their cover ratings. Blogpost on the analysis http://liveatthewitchtrials.blogspot.ie/2016/04/can-you-judge-book-by-its-cover.html Original dataset https://www.reddit.com/r/datasets/comments/3jiu4m/dataset_judgey_700k_ratings_for_100_book_covers/

CoverScatterplot.r

mydata =  read.csv("output.csv", encoding="UTF-8", header=TRUE)
#book_id	title	rating	num_ratings	num_reviews	book_id	x.x	x.y	cover_rating


attach(mydata)
plot(rating,num_reviews,main="Number of Ratings Number Reviews")

cor(num_ratings,num_reviews)
> cor(num_ratings,num_reviews)
[1] 0.9597442

 > cor(rating,cover_rating)
[1] 0.1609114
cor(rating,num_ratings)
cor(rating,num_reviews)
cor(num_ratings,cover_rating)
cor(num_reviews,cover_rating)
> cor(rating,cover_rating)
[1] 0.1609114
> cor(rating,cover_rating)
[1] 0.1609114
> cor(book_id,book_id2)
[1] 1
> cor(rating,num_ratings)
[1] 0.2141307
> cor(rating,num_reviews)
[1] 0.2658916
> cor(num_ratings,cover_rating)
[1] 0.3059627
> cor(num_reviews,cover_rating)
[1] 0.3307553
plot(rating,cover_rating,main="Judging a book by its cover")
library(ggplot2)

p1 <- ggplot(mydata, aes(x = rating, y = cover_rating))
p1 <- p1 + labs(x="Avg Cover Rating", 
                y = "Avg Book Rating") 

p1 <- p1 + ggtitle("Can you Judge a Book by its Cover?") + 
     theme(plot.title = element_text(lineheight=.8, face="bold"))
p1 <-p1 + annotate("text", x = 3, y = 1.5, label = "Correlation book and cover ratings is 0.16")
p1 <-p1 + geom_point()
p1
ggsave("Cover.png", width=10, height=10, dpi=300)

GoodReadsScraper.rb

require 'open-uri'
require 'nokogiri'
array = [1301,2165,2657,5246,7672,10210,14497,18342,19501,22034,22628,22635,33600,41865,43035,80467,105578,201114,297673,405784,439965,468328,469410,471381,505576,581332,825139,898885,1576816,1850833,1868343,1971304,1988626,2767052,3189111,3565645,4312691,4922079,5632446,6263078,6442769,6609765,6691280,6837103,6883046,7031817,7381740,8135213,8331609,8491980,8694331,8697395,9627849,9661374,9778945,10308500,10644930,10866398,12032357,12291438,12875355,13449852,13579626,13589182,15793049,15797397,15994634,16131077,16158542,16278318,17255444,17333230,17737039,18050053,18077769,18225810,18296030,18302455,18310201,18652002,18667945,19542841,20263206,20575425,20706269,20821614,20892558,20897517,20980987,21416678,21944886,22232035,22465597,22674531,22822858,23014725,23214378,23492682,23747672,24490083]
	array.each { |x| 
	# missing 22635,22034,22628
	print x.to_s+", "
	doc = Nokogiri::HTML(open("https://www.goodreads.com/book/show/"+(x).to_s))
	doc.xpath('//h1').each do |node|
		node=node.to_str.strip
      	print node+", "
    end
    #<span class="average" itemprop="ratingValue">4.23</span>
    ratings = doc.css('.itemprop')
	ratings.each do |rating|
  	print  rating.text+", "
	end
	#<span class="average" itemprop="ratingValue">4.23</span>
	ratings = doc.css('.average')
	ratings.each do |rating|
  	print  rating.text+", "
	end
	#<span class="value-title" title="62333" itemprop="ratingCount">62,333 Ratings</span>
	counts = doc.css('.value-title')
	counts.each do |count|
  	print count.text+", "
	end
	puts ""
	sleep(5)

}

output.csv

RFileMangling

library(dplyr)
#The group_by dplyr
mydata =  read.csv("dataset1.csv", encoding="UTF-8", header=TRUE)
#   book_id ratingval ratingcount
#get average rating for each book
summarise(group_by(mydata, book_id), mean(ratingval, na.rm = TRUE))

mydata <- mutate(mydata,total = ratingval*ratingcount)
mydata2 <- group_by(mydata, book_id)
summarise(mydata,total)
#sum the total votes for each book
totalP<-aggregate(mydata$total, by=list(book_id=mydata$book_id), FUN=sum)

#count the total number of votes for each book
totalC<-aggregate(mydata$ratingcount, by=list(book_id=mydata$book_id), FUN=sum)

(m1 <- merge(totalC, totalP, by.x = "book_id", by.y = "book_id"))
#get average rating for a cover

mydata <- mutate(m1,avg = x.y/x.x)

write.table(m1, file = "foo.csv", sep = ",", col.names = NA,
            qmethod = "double")