Gopher directory mirroring with pituophis

gophermirror.py

import pituophis
import os
from time import sleep

menus_to_download = ['gopher://']  # put your starting menu url here
sleep_between_downloads_sec = 0.1

def download_menu(url):
    req = pituophis.parse_url(url)
    text = req.get().text()
    with open('gophermap', 'w') as f:
        f.write(text)
    menu = pituophis.parse_menu(text)
    for i in menu:
        print(i.text)
        if (i.request().host == req.host) & (req.path in i.request().path):
            # checks that the item is 1. from the same host as the original and 2. in the same directory tree
            if i.request().type == '1':
                menus_to_download.append(i.request().url())
            elif i.request().type == 'i':
                pass
            elif i.request().path.startswith('URL:'):
                pass
            else:
                print('todownload', i.request().url())
                print(req.path, i.request().path)
                fn = '.' + i.request().path
                print('downloading to', fn)
                dir = '/'.join(fn.split('/')[:-1])
                if not os.path.exists(dir):
                    os.makedirs(dir)
                with open(fn, "wb") as f:
                    f.write(i.request().get().binary)
                sleep(sleep_between_downloads_sec)


while len(menus_to_download) > 0:
    download_menu(menus_to_download.pop(0))