Qfile

from qb import conf, rule, sh, Job, Reject, AWOL

from datetime import datetime, timedelta

window_re = r'(\d+s)'
type_re   = r'(bri|bid)'
date_re   = r'(\d{4}-\d{2}-\d{2})'

def timerange(start, stop=None, step=timedelta(days=1)):
    while start < stop:
        yield start
        start += step

def parse_date(datestr, fmt='%Y-%m-%d'):
    return datetime.strptime(datestr, fmt)

def dates(start, stop, fmt='%Y-%m-%d'):
    for date in timerange(parse_date(start), parse_date(stop)):
        yield date.strftime(fmt)

def ddate(date, ifmt='%Y-%m-%d', ofmt='%Y/%m/%d'):
    return datetime.strptime(date, ifmt).strftime(ofmt)

def today():
    t = datetime.now()
    return '%04d-%02d-%02d:%04d-%02d-%02d' % (t.year, t.month, t.day, t.year, t.month, t.day + 1)

def traildbs_per_day(window, type, start, stop):
    return ['%s/%s/%s.tdb' % (window, type, day) for day in dates(start, stop)]

def logs_per_day(window, type, date, pre='s3://adroll-us-west-2-logs'):
    return ['%s/logs/%s/%s' % (pre, ddate(date), type)]

@rule(r'default', '100s/bid/%s.stats' % today())
def do(*_):
    pass

@rule(r'%s/%s/%s:%s.stats' % (window_re, type_re, date_re, date_re), traildbs_per_day)
def do_stats(target, deps):
    if sh(('echo', 'stats!!!', target, ','.join(deps))).wait():
        raise AWOL(deps.items()[0]) # e.g.

@rule(r'%s/%s/%s.tdb' % (window_re, type_re, date_re), logs_per_day)
def do_traildb(target, deps):
    if sh(('echo', 'traildb!', target, ','.join(deps))).wait():
        raise Reject("traildb failed") # e.g.

@rule(r's3://adroll-us-west-2-logs/.*')
def do(*_):
    pass

@conf('nodes')
def nodes(*_):
    return [('admin@localhost', 8)]

if conf['profile'] == 'dist':
    conf.jobspace('s3://adroll-redshift/breadcrumbs/qbtest')