Copy timestamped data from AWS S3 to HDFS in Python (boto3)

copy_s3_to_hdfs.py

import boto3
import botocore

def copy_s3_to_hdfs(client, job_flow_id, day):
  """Copy data from AWS S3 to HDFS. Assumes that the file name includes
  date followed by a timestamp.
  """
  return client_throttle(
    client.add_job_flow_steps,
    JobFlowId=job_flow_id,
    Steps=[
      {
        'Name': 'Copy from S3 to HDFS',
        'ActionOnFailure': 'TERMINATE_CLUSTER',
        'HadoopJarStep': {
          'Jar': '/usr/share/aws/emr/s3-dist-cp/lib/s3-dist-cp.jar',
          'Args': [
            '--s3Endpoint',
            's3.amazonaws.com',
            '--src',
            's3n://path/to/my/data',  # this should be passed in as an argument
            '--srcPattern',
            '.*prefix.' + day.strftime(PATTERN_DATE_FORMAT) + '.([0-9]+.[0-9]+.[0-9]+).*',
            '--dest',
            'hdfs:///path/to/hdfs/input'
          ]
        }
      }
    ]