Ściąganie danych - wyniki konkursów NCN

scraping_ncn.R

library(httr)
library(XML)

#this is copied from my browser - thus Mozilla and Mac OS
headers <- ('{"Host":"www.ncn.gov.pl","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:54.0) Gecko/20100101 Firefox/54.0","Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","Accept-Language":"en-US,en;q=0.5","Accept-Encoding":"gzip, deflate, br","Content-Type":"application/x-www-form-urlencoded","Content-Length":"156","Referer":"https://www.ncn.gov.pl/statystyki/","Cookie":"has_js=1; __utma=198221488.1697332473.1495532703.1495532703.1495532703.1; __utmc=198221488; __utmz=198221488.1495532703.1.1.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided)","Connection":"keep-alive","Upgrade-Insecure-Requests":"1"}')
body <- list(zestawienie = "kwota", podzial = "wojewodztwo", 'zakres'='rok', 
						 konkursid = '', 'panelid'= '', typid = '', rokid= '1',
						 wojewodztwoid= '', plecid = '', tytulid=  '', rodzajejednostekid = '1')

granty <- vector(mode = "list", 6)
for(i in 1:6){
	body$rokid = i
	post <- POST("https://www.ncn.gov.pl/statystyki/", 
							 add_headers(headers),
							 body=body,
							 encode="form")
	doc.html = htmlTreeParse(content(post), useInternal = TRUE, encoding = "UTF-8")
	nazwy <- xpathApply(doc.html, '//table[@id="tabelawynik"]//th', xmlValue)
	kwoty <- xpathApply(doc.html, '//table[@id="tabelawynik"]//td', xmlValue)
	granty[[i]] <- data.frame(rok = 2010 +i,
														nazwa = tail(unlist(nazwy), -2), 
														kwota = as.numeric(gsub(" ", "", head(unlist(kwoty), -2))))
}
granty <- do.call(rbind, granty)