Databricks Recursive listing

recursive_listing.py

import boto3
s3 = boto3.client('s3')
bucket = 'databricks-datasets-virginia'
prefix = 'airlines/'
objects = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, Delimiter = "/")
files = []
for obj in objects['Contents']:
  files.append(obj)
while objects['IsTruncated']:
  continuation_key = objects['NextContinuationToken']
  objects = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, Delimiter="/", ContinuationToken=continuation_key)
  for obj in objects['Contents']:
      files.append(obj)
  
for file in files:
  print(file)