get_msgpack_from_ftp.py

import ftplib
import io
import os
import pprint
import sys
import time

import msgpack

DEFAULT_FTP_PORT = 5000

SERVER_MSGPACK_DIR = "/prepo/"
LOCAL_MSGPACK_DIR_NAME = "msgpack"
LOCAL_EXPORT_DIR_NAME = "export"

if __name__ == "__main__":
    ftp_addr = sys.argv[1]
    out_dir = sys.argv[2]

    # may be used as args
    delete_from_server = True
    keep_msgpack = True

    # prep dirs
    cur_export_dir = os.path.join(out_dir, f"{LOCAL_EXPORT_DIR_NAME}_{time.strftime('%Y%m%d_%H%M%S')}")
    cur_msgpack_dir = os.path.join(out_dir, f"{LOCAL_MSGPACK_DIR_NAME}_{time.strftime('%Y%m%d_%H%M%S')}")
    os.mkdir(cur_export_dir)
    if keep_msgpack:
        os.mkdir(cur_msgpack_dir)

    # start
    ftp = ftplib.FTP()
    ftp.connect(ftp_addr, DEFAULT_FTP_PORT)
    print(ftp.getwelcome())

    for name, facts in ftp.mlsd(path=SERVER_MSGPACK_DIR):
        if facts["type"] != "file":
            continue

        print(name)

        # get file
        cur_buffer = io.BytesIO()
        file_path = SERVER_MSGPACK_DIR + name
        ftp.retrbinary(f"RETR {file_path}", cur_buffer.write)
        cur_msgpack_content = cur_buffer.getbuffer().tobytes()
        if keep_msgpack:
            with open(os.path.join(cur_msgpack_dir, name), 'wb') as cur_out_file:
                cur_out_file.write(cur_msgpack_content)

        # try with removing tail bytes as it seems like sometimes there's garbage in the end
        cur_dict = None
        remove_tail_len = 0
        while cur_dict is None:
            try:
                cur_dict = msgpack.unpackb(cur_msgpack_content[:len(cur_msgpack_content) - remove_tail_len])
            except (msgpack.exceptions.ExtraData, ValueError):  #
                remove_tail_len += 1

        # write export
        print(cur_dict)
        with open(os.path.join(cur_export_dir, os.path.splitext(name)[0] + ".py"), 'w',
                  encoding='UTF-8') as cur_export_file:
            pprint.pprint(cur_dict, stream=cur_export_file)

        # clean up
        if delete_from_server:
            ftp.delete(file_path)

    ftp.quit()