Python Crawling Example

simple-crawler.py

import re
import ssl
from urllib import request

seedlist = ['https://scrapy.org/']

def extract_urls(url):
    """
    this function is used to extract URLs from HTML
    """
    results = []
    with request.urlopen(url, context=ssl._create_unverified_context()) as response:
        html = str(response.read())
        for candidate in re.findall(r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+", html):
            results.append(candidate)
    return results

while len(seedlist) > 0:
    url = seedlist.pop()
    extracted_links = extract_urls(url)
    print(extracted_links)
    seedlist.extend(extracted_links)