dashb-nusa-scrap.py

import urllib2
from bs4 import BeautifulSoup
from time import sleep
import csv
import sys
import json
import ssl
import time
from pymongo import MongoClient

client = MongoClient('localhost', 27017)
db = client.lokadata
panel = db.lokadata

reload(sys)
sys.setdefaultencoding('utf8')

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
def ambil(url):
	headers = { 'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2403.107 Safari/537.36','X-API-User-Token':' ' }
	req = urllib2.Request(url, None, headers)
	res = urllib2.urlopen(req, context=ctx)
	return json.loads(res.read())

url='https://data-api.lokadata.id/panel/area/'
lineList = [line.rstrip('\n') for line in open('x_idsiap.txt')]
#lineList.reverse()
for i in lineList:
	if panel.find({'id':i}).count() < 1:
		start = time.time()
		y={}
		y['id'] = i
		y['data'] = ambil(url+str(i))
		panel.insert_one(y)
		end =time.time()
		print('panel %s sudah diinput dalam %s detik' % (str(i), str(end-start)))
		lineList.pop(0)
	else:
		pass
		lineList.pop(0)


for i in line:
	start = time.time()
	y={}
	y['id'] = i
	y['data'] = ambil(url+str(i))
	panel.insert_one(y)
	end =time.time()
	print('panel %s sudah diinput dalam %s detik' % (str(i), str(end-start)))