Python script to run dated pig jobs

backfill.py

#!/usr/bin/env python
# encoding: utf-8
import sys
import getopt
from datetime import datetime, timedelta
from subprocess import call

help_message = '''
    -h, --help
    -s, --start_date <start_date>
    -e, --end_date <end_date>
    -f, --date_format <date_format>
    -c, --command <command>
    --use_date
    --increment <number_of_days>
'''


class Usage(Exception):
    def __init__(self, msg):
        self.msg = msg
        print 'Usage: '+sys.argv[0]+' -i <file1> [option]'
        

def main(argv=None):
    if argv is None:
        argv = sys.argv
    try:
        try:
            opts, args = getopt.getopt(argv[1:], "hs:e:f:c:v:i:", ["help", "start_date=", "end_date=", "date_format=", "command=", "use_date", "increment="])
        except getopt.error, msg:
            raise Usage(msg)
        
        start_date = None
        end_date = None
        date_format = "%Y%m%d"
        command = None
        use_date = False
        increment = 1
        # option processing
        for option, value in opts:
            if option == "-v":
                verbose = True
            if option in ("-h", "--help"):
                raise Usage(help_message)
            if option in ("-s", "--start_date"):
                start_date = value
            if option in ("-e", "--end_date"):
                end_date = value
            if option in ("-f", "--date_format"):
                date_format = value
            if option in ("-c", "--command"):
                command = value
            if option == "--use_date":
                use_date = True
            if option == "--increment":
                increment = int(value)
        
        d = datetime.strptime(start_date, date_format)
        edt = datetime.strptime(end_date, date_format)
        incr = timedelta(days=increment)
        while d <= edt:
            cur_command = ""
            if use_date:
                cur_command = command.replace("%%date%%", datetime.strftime(d, date_format))
            else:
                cur_command = command.replace("%%start_date%%", datetime.strftime(d, date_format))
            d += incr
            if not use_date:
                if d <= edt:
                     cur_command = cur_command.replace("%%end_date%%", datetime.strftime(d, date_format))
                else:
                     cur_command = cur_command.replace("%%end_date%%", datetime.strftime(edt, date_format))
            retcode = call(cur_command, shell=True)
            retstatus = "success" if retcode == 0 else "failed"
            print "%s [%s]" % (cur_command, retstatus)
    except Usage, err:
        print >> sys.stderr, sys.argv[0].split("/")[-1] + ": " + str(err.msg)
        print >> sys.stderr, "    for help use --help"
        return 2


if __name__ == "__main__":
    sys.exit(main())