Testing out the speed of the borg chunker with default chunker params.

borg_chunker_test.py

import os
import time
from borg.chunker import Chunker

CHUNK_MIN_EXP = 19  # 2**19 == 512kiB
CHUNK_MAX_EXP = 23  # 2**23 == 8MiB
HASH_WINDOW_SIZE = 0xfff  # 4095B
HASH_MASK_BITS = 21  # results in ~2MiB chunks statistically

CHUNKER_PARAMS = (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE)

seed = 1849058162

chunker = Chunker(seed, *CHUNKER_PARAMS)

flags_normal = os.O_RDONLY | getattr(os, 'O_BINARY', 0)
flags_noatime = flags_normal | getattr(os, 'O_NOATIME', 0)
fh = os.open("/Users/mikecampbell/Desktop/VMs/IE10 - Win7.ova", flags_noatime)

fd = os.fdopen(fh, 'rb')

chunks_count = 0
t0 = time.time()
for data in chunker.chunkify(fd, fh):
  chunks_count += 1

t1 = time.time()
print(chunks_count)
print(t1 - t0)

# 3799411200 bits in 15.18s
# 250290593 b/s
# 238 MB/s