coordinating-ecs-worker-v2.py

def worker_ftp(name, host, directory, bucket, input, output):
    pipeline = Pipeline(name=name, steps=[
        FtpDownload(host=host, directory=directory),
        S3Upload(bucket=bucket, key=output, chunksize=128*1024*1024)
    ])

    pipeline.start(input=input)

def worker_json(name, bucket, input, output):
    pipeline = Pipeline(name=name, steps=[
        S3Download(bucket=bucket),
        Ungzip(),
        XmlToJson(rowtag='logitem'),
        S3Upload(bucket=bucket, key=output, chunksize=128*1024*1024)
    ])

    pipeline.start(input=input)