ngambil dataset dari data.bandung.go.id

scrapDatasetBdg.py

import urllib
import json
from urllib2 import urlopen

import os
import glob
import csv
from xlsxwriter.workbook import Workbook

'''
Sebelum menjalankan aplikasi ini, pastikan library xlsxwriter sudah diinstall.
Cara installnya bisa dilihat di: https://xlsxwriter.readthedocs.io/
'''

def readJson():
	url = "http://data.bandung.go.id/api/3/action/package_list"
	response = urlopen(url)
	result = str(response.read())
	response.close

	bf = open("dataset.txt", "w")
	bf.write(result)
	bf.close

def readData():
	with open("dataset.txt") as datafile:
		data = json.load(datafile)

	getAllData(data["result"]) # kasih komen kalau mau cuma sebagian data yang diambil

	# kalau ingin download sebanyak xx file, hilangkan tanda # di bawah dan ganti range-nya:
	# xrange(awal, akhir)
	# lalu fungsi getAllData di atas dikasih #

	# for i in xrange(0,3):
	# 	d = data["result"][i]
	# 	u = "http://data.bandung.go.id/api/3/action/package_show?id=%s" %d
	# 	readDetailJson(u, d)

def getAllData(data):
	i = 0
	for d in data:
		i += 1
		u = "http://data.bandung.go.id/api/3/action/package_show?id=%s" %d
		print i,
		readDetailJson(u, d)

def readDetailJson(url, id):
	response = urlopen(url)
	result = response.read()
	data = json.loads(result)
	link = data["result"]["resources"][0]["url"]
	p = "hasil/%s.csv" %id
	print "lagi unduh %s" %link
	urllib.urlretrieve(link, p)

def convertToXlsx():
	for csvfile in glob.glob(os.path.join('.', 'hasil/*.csv')):
	    workbook = Workbook(csvfile + '.xlsx')
	    worksheet = workbook.add_worksheet()
	    with open(csvfile, 'rb') as f:
	        reader = csv.reader(f)
	        for r, row in enumerate(reader):
	            for c, col in enumerate(row):
	                worksheet.write(r, c, col)
	    workbook.close()

def print_menu():
	print 30 * "-" , "MENU" , 30 * "-"
	print "1. Ambil Dataset, simpan di dataset.txt, lalu ekspor ke csv dan xls"
	print "2. Ambil file csv saja dari dataset.txt"
	print "3. Keluar"
	print 67 * "-"

loop=True

while loop:
	print_menu()
	pilih = input("Mau pilih nomor berapa [1-4]? ")

	if pilih == 1:
		print "Kamu mau ambil dataset.txt dan ambil berkas csv nya"
		readJson()
		readData()
		convertToXlsx()
	elif pilih== 2:
		print "Kamu mau ambil file csv dari dataset.txt"
		fileada = os.path.isfile("datatext.txt")
		if fileada:
			readData()
		else:
			readJson()
			readData()
	elif pilih == 3:
		loop=False
	else:
		raw_input("yang bener aja bro.. Pilih yang bener: ")