-
-
Save orthodoc/11136923 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
####Download the data from Government of India open data portal##### | |
w_dir = getwd() | |
source(file=file.path(w_dir,"Code/Core.R")) | |
checkAndDownload(c("XML","RCurl","RJSONIO","plyr")) | |
### Alternative - 1: Using APIs ### | |
#JSON# | |
getJSONDoc <- function(link, res_id, api_key, offset, no_elements){ | |
jsonURL = paste(link, | |
"resource_id=",res_id, | |
"&api-key=",api_key, | |
"&offset=",offset, | |
"&limit=",no_elements, | |
sep="") | |
print(jsonURL) | |
doc = getURL(jsonURL) | |
fromJSON(doc) | |
} | |
getFieldNames <- function(t){ | |
#t: list | |
names(t[[4]]) | |
} | |
getCount <- function(t){ | |
#t: list | |
t[[3]] | |
} | |
getFieldType<-function(t){ | |
t[[4]] | |
} | |
getData <- function(t){ | |
t[[5]] | |
} | |
toDataFrame <- function(lst_elmnt){ | |
as.data.frame(t(unlist(lst_elmnt)), stringsAsFactors = FALSE) | |
} | |
acquire_x_data <- function(x,res_id,api_key){ | |
currentItr = 0 | |
returnCount = 1 | |
while(returnCount>0){ | |
JSONList = getJSONDoc(link="http://data.gov.in/api/datastore/resource.json?", | |
res_id=res_id, | |
api_key=api_key, | |
offset=currentItr, | |
no_elements=100) | |
DataStage1 = ldply(getData(JSONList),toDataFrame) | |
print(currentItr) | |
print(is(DataStage1$id)) | |
returnCount = getCount(JSONList) | |
if(currentItr == 0) { | |
returnData = DataStage1 | |
returnFieldType = ldply(getFieldType(JSONList),toDataFrame) | |
} | |
else if(returnCount > 0) returnData = rbind(returnData, DataStage1) | |
print(currentItr) | |
print(is(returnData$id)) | |
currentItr = currentItr + 1 | |
} | |
list(returnData,returnFieldType) | |
} | |
#get the resource list file | |
#(it has resource names and resource ids used for the API call) | |
resourceList = read.table( | |
file=file.path(w_dir,"Data/goi_api_resource_details.csv"), | |
header=TRUE, | |
sep=",", | |
as.is=TRUE) | |
api_key = read.table( | |
file=file.path(w_dir,"Data/goi_api_key_do_not_share.csv"), | |
header=TRUE, | |
sep=",", | |
as.is=TRUE) | |
#make the API call | |
res = subset(resourceList, resource_name == "pincode") | |
pincodeDetails = acquire_x_data(x = res[1], res_id = res[2], api_key = api_key) | |
save(pincodeDetails, file=file.path(w_dir,"Data/pincodeDetails.RData")) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment