Search in each of the s3 bucket and see if the given folder exists
#Author : Avinash Dalvi | |
#Reference question : https://stackoverflow.com/questions/62158664/search-in-each-of-the-s3-bucket-and-see-if-the-given-folder-exists/62160218#62160218 | |
import boto3 | |
client = boto3.client('s3') | |
bucket_name = "bucket_name" | |
prefix = "" | |
s3 = boto3.client("s3") | |
result = client.list_objects(Bucket=bucket_name, Delimiter='/') | |
for obj in result.get('CommonPrefixes'): | |
prefix = obj.get('Prefix') | |
file_list = ListFiles(client,bucket_name,prefix) | |
for file in file_list: | |
if "processed/files" in file: | |
print("Found",file) | |
def ListFiles(client, bucket_name, prefix): | |
_BUCKET_NAME = bucket_name | |
_PREFIX = prefix | |
"""List files in specific S3 URL""" | |
response = client.list_objects(Bucket=_BUCKET_NAME, Prefix=_PREFIX) | |
for content in response.get('Contents', []): | |
#print(content) | |
yield content.get('Key') |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment