s3 list paginator tricks.

s3_utils.py

import boto3

s3_client = boto3.client('s3')                                                                                                                                                      
                                                                                                                                                                                     
                                                                                                                                                                                     
def list_dirs(bucket, prefix):                                                                                                                                                       
    """ Yield direct child folders of the given prefix.                                                                                                                              
    """                                                                                                                                                                              
    if not prefix.endswith('/'):                                                                                                                                                     
        prefix += '/'                                                                                                                                                                
                                                                                                                                                                                     
    paginator = s3_client.get_paginator('list_objects_v2')                                                                                                                           
    results = paginator.paginate(Bucket=bucket, Prefix=prefix, Delimiter='/')                                                                                                        
                                                                                                                                                                                     
    for result in results:                                                                                                                                                           
        for prefix in result.get('CommonPrefixes', []):                                                                                                                              
            # Prefixes look like "<prefix>/<subdir>/"                                                                                                                                
            # This code replaces "<prefix>/" with an empty                                                                                                                           
            # space leaving "<subdir>" from the common prefix.                                                                                                                       
            yield prefix['Prefix'].replace(prefix, '', 1).strip('/')
            
def list_s3_keys(bucket, prefix='', suffix=''):                                                                                                                                                                                                                           
     s3_client = boto3.client('s3')                                                                                                                                                                                                                                        
     params = {'Bucket': bucket}                                                                                                                                                                                                                                           
                                                                                                                                                                                                                                                                           
     if isinstance(prefix, str):                                                                                                                                                                                                                                           
         params['Prefix'] = prefix                                                                                                                                                                                                                                         
                                                                                                                                                                                                                                                                           
     paginator = s3_client.get_paginator('list_objects_v2')                                                                                                                                                                                                                
     for result in paginator.paginate(**params):                                                                                                                                                                                                                           
         for obj in result['Contents']:                                                                                                                                                                                                                                    
             key = obj['Key']                                                                                                                                                                                                                                              
             if key.startswith(prefix) and key.endswith(suffix):                                                                                                                                                                                                           
                 yield key