Skip to content

Instantly share code, notes, and snippets.

@theoparis
Created February 19, 2024 02:00
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save theoparis/22c5d2ccb1a6556af4eae777467a6b47 to your computer and use it in GitHub Desktop.
Save theoparis/22c5d2ccb1a6556af4eae777467a6b47 to your computer and use it in GitHub Desktop.
Git clone multithreaded progress bar with python and rich
import os
from concurrent.futures import ThreadPoolExecutor
import pygit2
from rich.console import Group
from rich.live import Live
from rich.progress import (BarColumn, DownloadColumn, Progress, TextColumn,
TimeRemainingColumn, TransferSpeedColumn)
class MyRemoteCallbacks(pygit2.RemoteCallbacks):
def __init__(self, progress_bar: Progress, task: int):
super().__init__()
self.progress_bar = progress_bar
self.task = task
def transfer_progress(self, stats):
self.progress_bar.update(self.task,
total=stats.total_objects,
completed=stats.received_objects)
repos = [
"https://git.theos.sh/mirrors/aws-lc",
"https://gitlab.kitware.com/cmake/cmake",
"https://github.com/llvm/llvm-project",
"https://git.theos.sh/mirrors/musl",
"https://git.theos.sh/mirrors/zlib-ng",
"https://git.theos.sh/mirrors/zstd", "https://github.com/python/cpython",
"https://github.com/NetworkConfiguration/dhcpcd",
"https://github.com/openssh/openssh-portable",
"https://github.com/landley/toybox", "https://github.com/theoparis/linux",
"https://github.com/u-boot/u-boot", "https://github.com/riscv/opensbi.git",
"https://github.com/bminor/binutils-gdb"
]
def clone_repository(repo_url: str, progress: Progress):
repo_name = repo_url.split("/")[-1].replace(".git", "")
repo_path = os.path.join("work", repo_name)
if not os.path.exists(repo_path):
task = progress.add_task(f"Cloning {repo_name}",
total=1,
repo_name=repo_name)
remote_callbacks = MyRemoteCallbacks(progress, task)
repo = pygit2.clone_repository(repo_url,
repo_path,
bare=False,
callbacks=remote_callbacks)
progress.update(task, completed=1)
progress.log(f"Cloned {repo_name} to {repo_path}")
progress.stop_task(task)
progress.remove_task(task)
progress_bars = [
Progress(TextColumn("Cloning [bold blue]{task.fields[repo_name]}"),
BarColumn(), DownloadColumn(), TransferSpeedColumn(),
TimeRemainingColumn()) for repo_url in repos
]
render_group = Group(*progress_bars)
with Live(render_group):
with ThreadPoolExecutor(max_workers=4) as executor:
list(executor.map(clone_repository, repos, render_group.renderables))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment