Programmatically List S3 contents

list.py

from __future__ import print_function
import boto3
import configargparse
import sys


# -----------------------------------------------------------------------------
# Process
# -----------------------------------------------------------------------------

def run(options):
    import json

    s3 = boto3.resource('s3')
    bucket = s3.Bucket(options.bucket)
    tokens = options.key.split('/')
    key = '/'.join(tokens[:-1])

    sys.stdout.write('Contents of {}/{}\n'.format(options.bucket, key))
    for x in bucket.objects.filter(Prefix=key):
        sys.stdout.write('\t{}\n'.format(x.key[len(key):]))
        sys.stdout.flush()

    # Clear the buffer
    sys.stdout.write("\n")
    sys.stdout.flush()


# -----------------------------------------------------------------------------
# Main
# -----------------------------------------------------------------------------


def build_arg_parser():
    p = configargparse.ArgParser(
        prog='list',
        description='list the contents of an S3 "folder"',
        ignore_unknown_config_file_keys=True,
        default_config_files=['./config.ini'],
        args_for_setting_config_path=['-c', '--config'],
        args_for_writing_out_config_file=['--save-config']
    )
    p.add('--dump-config', action='store_true', dest='dump_config',
          help='dump config vars and their source')
    group = p.add_argument_group('S3')
    group.add('--s3-bucket', '-b', dest='bucket',
              required=True, env_var='S3_BUCKET',
              help='The S3 bucket that contains the data')
    group.add('--s3-key', '-k', dest='key',
              required=True, env_var='S3_KEY',
              help='The S3 path to the data')
    return p


if __name__ == '__main__':
    p = build_arg_parser()
    cfg = p.parse_args()

    if cfg.dump_config:
        print(p.format_values())

    run(cfg)