files

files.py

from collections import deque
import bs4
import requests
def extract_files(site_name):
    dirs = deque('/')
    result = []
    while len(dirs):
        path = dirs.pop()
        page = bs4.BeautifulSoup(requests.get(site_name + path).text, 'html.parser')
        for link in page.find_all('a'):
            href = link['href']
            assert not href.startswith('/')
            if href.startswith('..'): continue
            if href.endswith('/'): dirs.append(path + href)
            else: result.append(path + href)
        print(path)
    return result