A script to download wave files from an FTP server and put CartChunk data on them for ingestion. Restart partial downloads. Uses a breaker pattern and a progress bar. Apologies for the global vars.

getYBYG.py

import cdpwavefile
import ftplib
import socket
from datetime import date, timedelta
import os
import io
import sys
import time
import pyprind

Debug = False

Use_colors = os.environ['colorterm']

class Breaker(object):
    #Breaker pattern

    def __init__(self, status="closed", limit=3, delay=0):
        self.closed = True
        self.hit_counter = 0
        self.hit_limit = limit
        self.delay = delay

    def hit(self):
        self.hit_counter += 1
        if self.delay > 0:
            time.sleep(self.delay)
        if self.hit_counter > self.hit_limit:
            self.open()

    def open(self):
        self.closed = False

    def close(self):
        self.closed = True
        self.hit_counter = 0

class file_download(object):
    #A simple file object to help track progress

    def __init__(self, filename, total_size, ftp, block_size=None):
        #Does the file already exist?
        try:
            file_stat = os.stat(filename + ".part")
        except:
            self._file = io.open(filename + ".part", "wb")
            self.now_size = 0
        else:
            self._file = io.open(filename + ".part", "ab")
            self.now_size = file_stat.st_size
        finally:
            self._filename = filename
            self.total_size = total_size
            self.block_size = block_size
            self._ftp = ftp
            self.bar = None

    def write(self, data):
        delay = 1 - (len(data) / float(self.block_size))
        global Debug
        if Debug:
            print "* ", len(data), "* ", delay
        self._file.write(data)
        #print "*", self.now_size
        if self.bar is None:
            self.bar = pyprind.ProgBar(self.total_size)
            self.bar.update(iterations=self.now_size)
        self.bar.update(iterations=len(data))
        self.now_size = self.now_size + len(data)
        #if self.now_size > 10000000:
        #    self._ftp.sendcmd("NOOP")
        #time.sleep(1.5 * delay)

    def close(self):
        self._file.close()
        file_stat = os.stat(self._filename + ".part")
        if int(file_stat.st_size) == int(self.total_size):
            os.rename(self._filename + ".part", self._filename)
        else:
            print "File download failed"
            print "File size is supposed to be", self.total_size
            print "Downloaded file is", file_stat.st_size
            os.rename(self._filename + ".part", self._filename + ".fail")

def main():
    global Use_colors

    now = date.today()
    target_date = now - timedelta(days=now.weekday()-1)
    str_target_date = target_date.strftime('%y%m%d')
    print str_target_date
    end_date = target_date + timedelta(days=6)
    str_start_date = target_date.strftime('%Y/%m/%d')
    print str_start_date
    str_end_date = end_date.strftime('%Y/%m/%d')
    print str_end_date
    file_list = []
    meta_dict = {'yb1':{'title':'You Bet Your Garden SGMT 1',
                     'cutid':'49021'},
                 'yb2':{'title':'You Bet Your Garden SGMT 2',
                     'cutid':"49022"},
                 'yb3':{'title':'You Bet Your Garden SGMT 3',
                     'cutid':"49023"},
                 'yb4':{'title':'You Bet Your Garden SGMT 4',
                     'cutid':"49024"},
                 'ybp':{'title':'You Get Your Garden Promo',
                     'cutid':"49025"}
                 }

    try:
        ftp = establish_cnt()
    except:
        #try one more time
        ftp = establish_cnt()
    #ftp.cwd(str_target_date)
    ftp.retrlines('LIST *.wav', lambda n: parse_dir_list(n, file_list))
    for _file, size in file_list:
        if os.path.isfile(_file) is False:
            dl = file_download(_file, size, ftp, 209715)
            print "Downloading", _file
            sys.stdout.flush()
            dl_breaker = Breaker(limit=4, delay=5)
            while dl_breaker.closed:
                try:
                    ftp.retrbinary('RETR ' + _file, dl.write, dl.block_size,
                                   dl.now_size)
                except ftplib.error_temp as e:
                    print "Fooey, ftp error", e
                    ftp.close()
                    dl_breaker.hit()
                    ftp = establish_cnt()
                except socket.timeout as e:
                    if Use_colors:
                        print "\x1b[1;31m"
                    print "Socket timed out", e
                    if Use_colors:
                        print "\x1b[m"
                    ftp.close()
                    dl_breaker.hit()
                    ftp = establish_cnt()
                except KeyboardInterrupt:
                    if Use_colors:
                        print "\x1b[1;31m"
                    print "User interrupted download"
                    if Use_colors:
                        print "\x1b[m"
                    global Debug
                    if Debug:
                        Debug = False
                    else:
                        Debug = True
                    ftp.close()
                    dl_breaker.hit()
                    ftp = establish_cnt()
                except Exception as e:
                    print "Other problem", e
                    ftp.close()
                    dl_breaker.hit()
                    ftp = establish_cnt(str_target_date)
                else:
                    dl_breaker.open()
            dl.close()
            sys.stdout.flush()
            time.sleep(2)

    ftp.close()

    print file_list
    #exit()
    #file_list = []
    for _file, size in file_list:
        mycdpfile = cdpwavefile.CDPFile()
        root_name = _file[:-4]
        segment = root_name[0:3]
        if segment in meta_dict.keys():
            print root_name, segment
            mycdpfile.cart.title = meta_dict[segment]['title'] + " " + str_target_date
            mycdpfile.cart.cutnum = meta_dict[segment]['cutid']
            mycdpfile.cart.startdate = str_start_date
            mycdpfile.cart.starttime = "00:00:00"
            mycdpfile.cart.enddate = str_end_date
            mycdpfile.cart.endtime = "23:59:59"
            mycdpfile.cart.tagtext = ""
            mycdpfile.cart.appid = "PythonScript"
            mycdpfile.ReadWaveFile(_file)
            mycdpfile.WritePCMWaveFile("x:\\CDautoload\\" + root_name + "_cart.wav")
            os.unlink(_file)

def parse_dir_list(data, file_list):
    tokens = data.split()
    file_list.append((tokens[3], tokens[2]))

def establish_cnt():
    ftp = ftplib.FTP(host='0.0.0.0',
              user='*********',
              passwd='********',
              timeout=5)
    ftp.set_debuglevel(2)
    #ftp.login()
    ftp.cwd("********/******")
    return ftp

if __name__ == "__main__":
    print sys.executable
    if 'pythonw' in sys.executable:
        with open("output.txt", "w") as sys.stdout:
            with open("error.txt", "w") as sys.stderr:
                main()
    else:
        main()