Skip to content

Instantly share code, notes, and snippets.

@nicolas17

nicolas17/fastdd.py

Created November 11, 2025 22:23
Show Gist options
  • Save nicolas17/6d3db01948e3ad02cd12a9686b3b4c7f to your computer and use it in GitHub Desktop.
Save nicolas17/6d3db01948e3ad02cd12a9686b3b4c7f to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. nicolas17 created this gist Nov 11, 2025.
    40 changes: 40 additions & 0 deletions fastdd.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,40 @@
    #!/usr/bin/python3

    # Copyright (c) 2024 Nicolás Alvarez
    # SPDX-License-Identifier: MIT

    # Usage: fastdd.py <source> <dest>
    #
    # Will copy blocks from source to destination,
    # unless the destination block already has the correct content.

    import sys
    import os
    from tqdm import tqdm

    BLOCK_SIZE=1024*4096
    data_written = 0
    data_saved = 0

    size = os.path.getsize(sys.argv[1])
    #size = 7969177600
    pbar = tqdm(total=size, unit="B", unit_scale=True)

    pos = 0
    with open(sys.argv[1], "rb") as inf, open(sys.argv[2], "r+b") as outf, pbar:
    while True:
    outf.seek(pos)
    block = inf.read(BLOCK_SIZE)
    if len(block) == 0: break

    if outf.read(BLOCK_SIZE) != block:
    outf.seek(pos)
    outf.write(block)
    data_written += len(block)
    else:
    data_saved += len(block)
    pbar.update(len(block))
    pos += len(block)

    print(f"Wrote {data_written/1024/1024:.1f}MB")
    print(f"Avoided writing {data_saved/1024/1024:.1f}MB")