Clone GoIndex recursively

goindex_clone.py

#
# Requires requests module.
# First argument is the URL to the target goindex- this one is necessary, obviously.
# Second argument is the folder of the name where the contents will be stored-
# defaults to 'goindex_download' if not provided.
# Other arguments will be ignored.
#
# Since my target had an unstable worker which throws 500 quite frequently,
# `request` method automatically retries the request until it receives 200.
# The delay is hardcoded to 1 second- you can change it if you want.
#

import os
import sys
import time
import requests

if len(sys.argv) < 2:
    print("Error: Site URL not provided")
    sys.exit(1)

grab_url = sys.argv[1]
name = sys.argv[2] if len(sys.argv) >= 3 else "goindex_download"

def request(url, method, stream=False):
    r = requests.request(method, url, stream=stream)
    if r.status_code != 200:
        print(f"Error: {r.status_code} while trying {url}. Retrying...")
        time.sleep(1)
        return request(url, method, stream)
    return r

def download_dir(url, name):
    print(f"Downloading folder '{name}'...")
    try:
        os.mkdir(name)
    except FileExistsError:
        pass
    os.chdir(name)
    url = url + "/" if not url.endswith("/") else url
    r = request(url, "POST")
    filelist = r.json()['files']
    for file in filelist:
        if file['mimeType'] == "application/vnd.google-apps.folder":
            download_dir(f"{url}{file['name']}", file['name'])
        else:
            download_file(f"{url}{file['name']}", file['name'], int(file['size']))
    os.chdir("..")

def download_file(url, name, size):
    print(f"Downloading file '{name}'...")
    bcount = 0
    r = request(url, "GET", True)
    with open(name, "wb") as f:
        for chunk in r.iter_content(4*1024*1024):
            bcount += f.write(chunk)
    print(bcount, size, sep="/", end="\r")
    print("\nFinished!")
    if bcount != size:
        print("Error: Size mismatch! redownloading...")
        return download_file(url, name, size)
    return bcount


download_dir(grab_url, name)