Script for downloading files from NOAA FTP links

noaa_downloader.py

#!/usr/bin/env python3

# Script for downloading files from NOAA ftp server and do necessary
# cleanups.

import os
import argparse
from ftplib import FTP
import tarfile
import glob

__author__ = 'Ashfaqur Rahman <sajib.finix@gmail.com>'
__version__ = '1.0.4'

parser = argparse.ArgumentParser(description="Script for downloading files from NOAA FTP links.")
parser.add_argument('--file', '-f', help="Specify a text file containing links in each line.")
parser.add_argument('links', nargs='*', help="NOAA FTP link. You can specify multiple links at once. Or use --file, -f option in case of multiple links")

args = parser.parse_args()

def untar(file_name):
    """
    Untar .tar files

    Params:
        file_name: str
            file name

    Returns:
        None
    """
    tarf = tarfile.open(file_name)
    tarf.extractall()

def cleanup(file_pattern):
    """
    Removes unnecessary files

    Args:
        file_pattern (str): pattern to serach files

    Returns:
        None
    """
    files = glob.glob(file_pattern)

    for file_name in files:
        token = file_name
        token = token[-8:-5]
        if int(token) > 75:
            os.remove(file_name)

def download_from_links(links):
    """
    Download files from ftp links

    Params:
        links: list
            list of links to download from.

    Returns:
        None
    """
    for link in links:
        index = link.rfind('pub')
        ftp_dir = link[index:]
        
        ftp_url = link.replace('ftp://', '')
        ftp_url = ftp_url.replace(ftp_dir, '')
        ftp_url = ftp_url.rstrip("/")
        
        ftp = FTP(ftp_url)
        ftp.login()
        ftp.cwd(ftp_dir)
        
        file_list = ftp.nlst()

        for file_name in file_list:
            # First part of the filename
            prem = file_name[:14]

            # Download the file
            print("Downloading %s" % file_name)
            fhandler = open(file_name, 'wb') 
            ftp.retrbinary('RETR %s' % file_name, fhandler.write)
            print("Complete!")

            # Untar file
            print("Untar %s" % file_name)
            untar(file_name)
            print("Complete!")

            print("Removing %s" % file_name)
            os.remove(file_name)
            print("Complete!")

            # Cleanup
            print("Removing unnecessary files...")
            cleanup(prem + '*')
            print("Done!")

download_from_links(args.links)