Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
two_progress_bars
def download_data(s3_keys: List[Tuple[str, int]], total_size: int):
size_progress = Progress(
"{task.description}",
TextColumn("[bold blue]{task.fields[key]}", justify="right"),
BarColumn(),
DownloadColumn(),
TransferSpeedColumn(),
TimeRemainingColumn(),
)
keys_progress = Progress("{task.description}", BarColumn(), "{task.completed} of {task.total}")
with size_progress, keys_progress:
size_task = size_progress.add_task("[red]Downloading", total=total_size, key='?')
keys_task = keys_progress.add_task("[blue]S3 Keys", total=len(s3_keys))
for key, file_size in s3_keys:
size_progress.update(size_task, advance=file_size, key=key)
keys_progress.advance(keys_task)
output = base_path.joinpath(*key.split(prefix)[-1].split('/'))
if output.exists() and output.read_bytes():
continue
output.parent.mkdir(parents=True, exist_ok=True)
output.touch()
s3.download_file(Bucket=bucket_name, Key=key, Filename=str(output))
console.log(f'Successfully downloaded [bold cyan]{len(s3_keys)}[/bold cyan] keys')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment