hk session 2.r

priv = read.csv("data/private_capital.csv")
pub = read.csv("data/public_capital.csv")

colnames(pub)[1] = "YEAR"

pub = pub[-1, ]

d = merge(priv, pub, by.x="Year", by.y="YEAR", all=T)

colnames(pub)[-1] = paste("pub", colnames(pub)[-1], sep = "_")
head(pub)
d = merge(priv, pub)
head(d)

library(reshape2)
priv = melt(priv, id.vars="Year")
colnames(priv)[-1] = c("country", "private.wealth")
head(priv)

pub = melt(pub, id.vars="Year")
colnames(pub)[-1] = c("country", "public.wealth")
head(pub)

d = merge(priv, pub)
head(d)

d$total.wealth = d$private.wealth + d$public.wealth
head(d)

w = dcast(d, Year ~ country, value.var="total.wealth")

head(d)

?mean

dcast(d, Year ~ ., value.var="total.wealth", fun.aggregate = mean, na.rm=T)

head(d)

d$decade = floor(d$Year / 10) * 10

dcast(d, decade ~ country, value.var="private.wealth", fun.aggregate = mean, na.rm=T)


head(d)

aggregate(d[c("private.wealth", "public.wealth")], d[c("decade", "country")], mean)




means = aggregate(d[c("private.wealth")], d[c("decade", "country")], mean)
colnames(means)[3] = "m"
sds = aggregate(d[c("private.wealth")], d[c("decade", "country")], sd)
colnames(sds)[3] = "sd"
head(sds)
head(means)
merge(sds, means)

mymean = function(x) {
  x = sort(x)
  x = x[c(-1, -length(x))]
  mean(x)
}

zscore = function(x) {
  (x - mean(x)) / sd(x)
}

x = c(1,5,6,7,99)
mymean(c(1,5,6,7,99))

nunique = function(x) length(unique(x))

nunique(c(1,2,2,2,2,2,2,3))

zscore(x)

aggregate(d[c("private.wealth")], d[c("decade", "country")], mymean)
colnames(sds)[3] = "sd"

dcast(d, decade ~ country, value.var="private.wealth", fun.aggregate = nunique)

load("api_auth.rda")


library(twitteR)
twitteR::setup_twitter_oauth(tw_consumer_key, tw_consumer_secret, tw_token, tw_token_secret)

tw = searchTwitteR("#bigdata", n=25)
t = plyr::ldply(tw, as.data.frame)


library(Rfacebook)
token = fbOAuth(fb_app_id, fb_app_secret)

save(token, x, p, file="/tmp/token.rda")

load("/tmp/token.rda")

saveRDS(token, "/tmp/token.rds")
fb_token = readRDS("/tmp/token.rds")



p = getPage("nytimes", token=token)
head(p)

Rfacebook::callAPI()
twitteR::getCurRateLimitInfo()
x  = 99

library(rtimes)
load("api_auth.rda")

options(nytimes_as_key=nyt_api_key)
nyt_api_key


res = as_search(q="trump", fq="section_name: front page", begin_date = 20160101, end_date = 20160201)
results = res$data
  
maxpage = floor(res$meta$hits / 10)
for (page in 1:maxpage) {
  message(page)
  res = as_search(q="trump", fq="section_name: front page", begin_date = 20160101, end_date = 20160201, page=page)
  results = c(results, res$data)
}
length(results)
r2 = as_search(q="trump", begin_date = 20160101, end_date = 20160201, page = 2)
r3 = as_search(q="trump", begin_date = 20160101, end_date = 20160201, page = 3)
r4 = as_search(q="trump", begin_date = 20160101, end_date = 20160201, page = 4)
r5 = as_search(q="trump", begin_date = 20160101, end_date = 20160201, page = 5)
r6 = as_search(q="trump", begin_date = 20160101, end_date = 20160201, page = 6)

class(r)
names(r)
r$data




query_times = function(token, q) {

  url = "https://api.nytimes.com/svc/search/v2/articlesearch.json"
  query = list('api-key'=token, q=q)

   res = httr::GET(url, query=query)
  if (status_code(res) != 200) stop("Wrong status: ", status_code(res))

  res = content(res)
  res$response$docs
}

query_times(nyt_api_key, "trump")

class(res)
names(res)
res$response$docs


  'q': "trump",
  'fq': "section_name: front page",
  'begin_date': "20160101",
  'end_date': "20160201"
});