Crawl method of the SitemapGenerator class for a tutorial on the SSaurel's Blog

crawl.py

def crawl(self, url, level):
  print("Level: " + str(level) + "/ Explore " + url)

  page = requests.get(url)

  if page.status_code == 200 :
      url =  urllib.parse.urldefrag(url)[0] # we don't add url with fragments

      if url not in self.urls :
          self.urls[url] = level

          soup = BeautifulSoup(page.content, "html.parser")

          for link in soup.findAll('a') :
              try :
                  href = link.get('href')
                  result = urlparse(href)
                  newurl = None

                  if result.hostname == None and href is not None:
                      # same domain
                      newurl = self.root + ("/", "")[href.startswith("/")] + href;

                  elif result.hostname == self.hostname :
                      newurl = href;

                  if newurl != None :
                      self.crawl(newurl, level + 1)


              except TypeError:
                  print("Error for link:" + link.get('href'))

      else :
          if self.urls[url] > level :
              self.urls[url] = level 

  else :
      print(url + " unreachable")