ihfazhillah/surahs_scraper.py

## surahs_scraper.py
#!/usr/bin/python3

"""Scrape alquran surahs into json format"""

import requests
from bs4 import BeautifulSoup
import json

URL = "http://www.islamology.com/quran/fahres.asp"
def get_response(url=URL):
    """get reponse object"""
    resp = requests.get(url)
    return resp

def parse_response(response):
    """parse response data into dict object
    dict(surah_no, surah_name, surah_juz, surah_page,
    surah_ayat, surah_type)"""
    soup = BeautifulSoup(response.content, 'html.parser')
    fahrasah = soup.select("tbody.Common_TBODY > tr")


    result = list()

    for f in list(fahrasah):
        data = f.select("td")
        d = dict(surah_no=int(data[0].text),
                 surah_name=data[1].text.strip(),
                 surah_juz=data[2].text.strip(),
                 surah_page=int(data[3].text),
                 surah_ayat=int(data[4].text),
                 surah_type=data[5].text.strip())
        result.append(d)

    return result


def dump_json(data, fn, indent=4):
    """dump data into json file"""
    with open(fn, "w") as f:
        json.dump(data, f, indent=indent, ensure_ascii=False)


if __name__ == '__main__':

    rsp = get_response()
    data = parse_response(rsp)
    dump_json(data, "daftar_surat.json")
	#!/usr/bin/python3

	"""Scrape alquran surahs into json format"""

	import requests
	from bs4 import BeautifulSoup
	import json

	URL = "http://www.islamology.com/quran/fahres.asp"
	def get_response(url=URL):
	"""get reponse object"""
	resp = requests.get(url)
	return resp

	def parse_response(response):
	"""parse response data into dict object
	dict(surah_no, surah_name, surah_juz, surah_page,
	surah_ayat, surah_type)"""
	soup = BeautifulSoup(response.content, 'html.parser')
	fahrasah = soup.select("tbody.Common_TBODY > tr")



	result = list()

	for f in list(fahrasah):
	data = f.select("td")
	d = dict(surah_no=int(data[0].text),
	surah_name=data[1].text.strip(),
	surah_juz=data[2].text.strip(),
	surah_page=int(data[3].text),
	surah_ayat=int(data[4].text),
	surah_type=data[5].text.strip())
	result.append(d)

	return result


	def dump_json(data, fn, indent=4):
	"""dump data into json file"""
	with open(fn, "w") as f:
	json.dump(data, f, indent=indent, ensure_ascii=False)


	if __name__ == '__main__':

	rsp = get_response()
	data = parse_response(rsp)
	dump_json(data, "daftar_surat.json")