nzDownload.py

import ftputil
import datetime
import os


def fileTypeMatch(name):
    return (name.endswith('.V2A') or name.endswith('.V1A') or name.endswith('.V3A'))

wantedDates = [
    #"2010-09-03 16:35:41"
    #"2013-08-16 05:31:16",
    "2007-12-20 07:55:16",
    "2011-02-22 00:04:19"

    ]
def getFiles():
    with ftputil.FTPHost("ftp.geonet.org.nz","anonymous","matt.d.clemens@gmail.com") as host:
        baseUrl = "/strong/processed/Proc/"
        host.keep_alive()
                
        for time in wantedDates:
            # Converting out wanted dates into a python Datetime object
            d = datetime.datetime.strptime(str(time),"%Y-%m-%d %H:%M:%S")
            # getFtpFilePath
            url = d.strftime("%Y/%m_%b/%Y-%m-%d_%H%M%S")
            # Local path and folder structure
            newpath = "NewZealand_"+d.strftime("%m%d%Y%H%M%S") +"/"
            # We only create a new folder if one doesn't exist. This avoids
            # us deleting our previous downloads.
            if not os.path.exists(newpath):
                os.makedirs(newpath)
                os.makedirs(os.path.join(newpath,"GNS_NZ"))
            try:
                host.chdir(baseUrl+ url)
                print("Found :" +baseUrl+url)
                for vol in ["/Vol1","/Vol2","/Vol3"]:
                    # Make sure we dont time out.
                    host.keep_alive()
                    print("Working on " + vol)
                    
                    # The v#A files live within the data dir of each Vol#
                    folderUrl = host.curdir+vol+"/data"
                    ftpFiles = host.listdir(folderUrl)
                    
                    #filter for only original files not backups or other data types we dont want.
                    ftpFiles = [n for n in ftpFiles if fileTypeMatch(n)]  
                    possible = len(ftpFiles)
                    
                    # get list of already downloaded files
                    localFiles = os.listdir(newpath)
                    
                    # We compare the lists to see what we still need to download.
                    stillToDownload = list(set(ftpFiles) - set(localFiles))
                    remaining = len(stillToDownload)
                    
                    print("Total Files to Download: "+str(possible))
                    print("Left to Download: "+ str(remaining))
                    for name in stillToDownload:
                        print("#"+ str(remaining)+" ["+vol+"]["+name+"]")
                        remaining -= 1
                        host.download(folderUrl + "/" + name, newpath + name)  # remote, local
                            
            except ftputil.error.PermanentError:
                # Couldn't find the requested earthquake
                print("Missing! :" +baseUrl+url) 
            except ftputil.error.TemporaryError:
                # If we still time out we just reconnect.
                print("Error: TimedOut. Quick Fix: Retry Program.")
                getFiles()
                #host = ftputil.FTPHost("ftp.geonet.org.nz","anonymous","matt.d.clemens@gmail.com")
                continue

getFiles()