ftp_to_rclone.py

# Made by Giuseppe La Malfa
# this script allows you to transfer an FTP directory's contents into an rclone remote
# run with python 3, and make sure you have correctly installed and configured lftp, sed, wget and rclone

# example usage:
# $ python ftp_to_rclone.py "http://archlinux.mirror.garr.it/archlinux/core/os/x86_64/" "test:arch"
# to kill script, press CTRL-C twice

import os
import subprocess
import threading
import sys
from pathlib import Path


def usage():
    print("usage:\npython ftp_rclone.py <ftp_directory> <rclone_dir>")
    exit(-1)


tmpPrefix = '.glitchedcode_tmp'
serverAddress = ""
remotePrefix = ""
downloadedFiles = []


def fix_slashes(arg):
    return arg.replace("//", "/").replace("\\/", "/")


def get_file_list():
    subprocess.run(["rm", "-f" "\"{tmpPrefix}/list_stage1.txt\""])
    os.system(
        f"mv \"{tmpPrefix}/list_stage2.txt\" \"{tmpPrefix}/list_stage2.old\"")
    os.system(
        f"lftp -e \"find;exit\" \"{serverAddress}\" > \"{tmpPrefix}/list_stage1.txt\"")
    os.system(
        f"sed -e 's/^.\///' \"{tmpPrefix}/list_stage1.txt\" >> \"{tmpPrefix}/list_stage2.txt\"")


def download_file(path):
    localPath = f"{tmpPrefix}/files/{path}".strip()
    webPath = path.replace(" ", "%20")
    ftpPath = f"{serverAddress}{webPath}".strip()

    # do the dir
    subprocess.run(["mkdir", "-p", localPath])
    subprocess.run(["rm", "-r", localPath])
    # download the file
    subprocess.run(["wget", ftpPath, "-O", localPath])


def upload_file(path):
    local_path = f"{tmpPrefix}/files/{path}".strip()
    local_path = fix_slashes(local_path)

    if not os.path.isfile(local_path):
        return

    print(f"uploading {path}")

    webPath = path.replace(" ", "%20")
    webPath = webPath.strip()

    tmpIndex = 0
    for i in range(len(path)-1, 0, -1):
        if path[i] == '/' or path[i] == '\\':
            tmpIndex = i
            break
    remote_path = f"{remotePrefix}/{path[:tmpIndex]}"
    remote_path = fix_slashes(remote_path)

    # sync to rclone remote
    with open(local_path) as file:
        pass
    subprocess.run(["rclone", "sync", local_path, remote_path])
    subprocess.run(["rm", "-f", local_path])

class upload_thread(threading.Thread):
    def __init__(self, semaphore):
        threading.Thread.__init__(self)
        self.semaphore = semaphore
        self.stopped = False

    def run(self):
        while True:
            self.semaphore.acquire()
            if self.stopped and len(downloadedFiles) == 0:
                return
            currentFile = downloadedFiles.pop(0)
            upload_file(currentFile)

    def stop(self):
        self.stop = True
        self.semaphore.release()


if __name__ == "__main__":
    if len(sys.argv) < 3:
        usage()

    serverAddress = sys.argv[1].strip()
    remotePrefix = sys.argv[2].strip()

    subprocess.run(["rm", "-rf", tmpPrefix+"/files"])
    subprocess.run(["rm", "-rf", tmpPrefix+"/list_stage1.txt"])
    os.system(f"mkdir -p \"{tmpPrefix}\"")
    print("Getting file list...")
    get_file_list()
    joined = False

    try:

        downloadedFilesSemaphore = threading.Semaphore(0)
        file_list = open(f"{tmpPrefix}/list_stage2.txt")
        ulthread = upload_thread(downloadedFilesSemaphore)
        ulthread.start()

        for line in file_list:
            download_file(line)
            downloadedFiles.append(line)
            downloadedFilesSemaphore.release()

        ulthread.stop()
        ulthread.join()

        joined = True

    except KeyboardInterrupt:
        if not joined:
            ulthread.stop()
            ulthread.join()
        subprocess.run(["rm", "-rf", tmpPrefix])
        raise

    subprocess.run(["rm", "-rf", tmpPrefix])
    exit(0)