Code to scrape RocketHub's xml feeds for the #SciFund projects, add it to a log file, then plot it. Run the data scrape without appending the first time to generate a log file.

projects.csv

scifundDataScrape.r

library(XML)
library(plyr)
##Rscript scifundDataScrape.r

#setwd("~/scifund_analysis")

thisDate<-Sys.time()
projects<-read.csv("./projects.csv")

#data frame of what's going on now
ndf<-ddply(projects, .(ID, Project.Link), function(adf){
    prj<-adf$Project.Link[1]
    dollaz <- xmlValue(xmlRoot(xmlTreeParse(paste(prj, "xml", sep="."), getDTD = F))[["raised_dollars"]])
    return(c(Funding=dollaz))

})


ndf$DateTime<-rep(thisDate, nrow(ndf))


write.table(ndf, "./log.csv", sep=",", row.names=F, append=TRUE, col.names = FALSE) #appends to pre-existing log file

#####write.table(ndf, "./log.csv", sep=",", row.names=F) #creates new log file, comment out after running once

scifundGraphMake.r

library(plyr)
library(ggplot2)
##Rscript scifundGraphMake.r

#setwd("~/scifund_analysis")
fullTab<-read.csv("./log.csv")
projects<-read.csv("./projects.csv")

maxGoal<-sum(projects$Goal)

####generate graphs


mdf<-merge(projects, fullTab)  #all
#mdf$Date<-strsplit(as.character(mdf$DateTime)," ")[[1]][1]

dateTemp<-strsplit(as.character(mdf$DateTime)," ")
mdf$Date<-sapply(dateTemp, function(x) x[1])

mdf_daily<-ddply(mdf, .(ID, Title, Name, Date), summarize, Funding=max(Funding, na.rm=T))

#mdf$DateTime<-as.POSIXct(as.character(mdf$DateTime), format="%y/%m/%d %H:%M:%S")
mdf$DateTime<-as.POSIXct(as.character(mdf$DateTime))
tdf<-ddply(mdf, .(DateTime), summarize, Date=Date[1], Funding=sum(Funding), Group=1) #total
tdf_daily<-ddply(tdf, .(Date), summarize, Funding=max(Funding, na.rm=T), Group=1) #total



##GENERATE GRAPHS
#setwd("~/webdocs/public_html/scifundGraphs")


#hourlies
png("1_all_projects_hourly.png")
qplot(DateTime, Funding, colour=Name, group=Name, data=mdf, geom=c("point", "line"), 
      main="All Projects", xlab="Date")+theme_bw(base_size=14)+opts(legend.position = "none") +
        scale_x_datetime(limits=c(as.POSIXct('2011-11-07 18:00:02 PST'), Sys.time()), format =  "%m-%d")

dev.off()

png("1a_all_projects_percent_hourly.png")
qplot(DateTime, Funding/Goal*100, colour=Name, group=Name, ylab="% of Goal", data=mdf, geom=c("point", "line"), 
      main="All Projects", xlab="")+theme_bw(base_size=14)+opts(legend.position = "none") +
        scale_x_datetime(limits=c(as.POSIXct('2011-11-07 18:00:02 PST'), Sys.time()), format =  "%Y-%m-%d")

dev.off()


png("3_total.png")
qplot(DateTime, Funding, main="Total #SciFund Donations\n", group=Group, data=tdf, 
      geom=c("point", "line"), lwd=I(1.5))+theme_bw(base_size=16)+xlab("\nDate")+ylab("Dollars Contributed\n")+
        scale_x_datetime(limits=c(as.POSIXct('2011-11-07 18:00:02 PST'), Sys.time()), format =  "%m-%d")

dev.off()

###all projects
d_ply(mdf, .(Name), function(adf) {
  PRJ<-adf$Title[1]
  png(paste(PRJ, "_hourly.png", sep=""))
  a<-qplot(DateTime, Funding, data=adf, geom=c("point", "line"), group=Name, main=PRJ)+
      theme_bw(base_size=14)+
        scale_x_datetime(limits=c(as.POSIXct('2011-11-07 18:00:02 PST'), Sys.time()), format =  "%Y-%m-%d")

  print(a)
  dev.off()  
})



#daylies
png("2_all_projects_daily.png")
qplot(Date, Funding, colour=Name, group=Name, data=mdf_daily, geom=c("point", "line"), main="All Projects")+
  theme_bw(base_size=14)+xlab("\nDate")+ylab("Dollars Contributed\n")
  opts(legend.position = "none")
dev.off()

png("4_total_daily.png")
qplot(Date, Funding, main="Total #SciFund Donations", group=Group, data=tdf_daily, geom=c("point", "line"))+theme_bw(base_size=16)+
	xlab("\nDate")+ylab("Dollars Contributed\n")
dev.off()



###all projects
d_ply(mdf_daily, .(Name), function(adf) {
  PRJ<-adf$Title[1]
  png(paste(PRJ, "_daily.png", sep=""))
  a<-qplot(Date, Funding, data=adf, geom=c("point", "line"), group=Name, main=PRJ)+theme_bw(base_size=14)+xlab("\nDate")+ylab("Dollars
 Contributed\n")
  print(a)
  dev.off()  
})