btrfs + beesd dedup setup on debian 10

README.txt

# install btrfs + bees (dedup daemon)
# ---------------------------------
 
# add new "big" disk to vm (edit vm, add new disk)
 
# be root
 
su -
 
# check for new disk in guest
 
tail /var/log/message | grep sd
 
# there should be something like:
#kernel: [2435321.073280] sd 0:0:1:0: Attached scsi generic sg2 type 0
#kernel: [2435321.074127] sd 0:0:1:0: [sdb] 10485760000 512-byte logical blocks: (5.37 TB/4.88 TiB)
#kernel: [2435321.074324] sd 0:0:1:0: [sdb] Write Protect is off
#kernel: [2435321.074396] sd 0:0:1:0: [sdb] Cache data unavailable
#kernel: [2435321.076362] sd 0:0:1:0: [sdb] Attached SCSI disk
 
# here we see that sdb got attached with about 5TB of space
 
# since debian has not the latest kernel installed,
# the community advices to install the latest kernel for latest btrfs security patches
# ref: https://oitibs.com/linux-kernel-5-4-on-debian-10/
 
echo "deb http://ftp.us.debian.org/debian/ buster-backports main" >> /etc/apt/sources.list
echo "deb-src http://ftp.us.debian.org/debian/ buster-backports main" >> /etc/apt/sources.list
apt update
update upgrade
apt install -t buster-backports linux-image-amd64
 
reboot
 
# login as root again
 
# check kernel version, should be (07-2020: 5.6...)
# if this is older than 5.3, then try to research why kernel uprade went wrong, ..
 
uname -r
 
# now we configure our btrfs
# https://www.tecmint.com/create-btrfs-filesystem-in-linux/
# https://www.howtogeek.com/howto/40702/how-to-manage-and-use-lvm-logical-volume-management-in-ubuntu/
 
# install fs tools we need
 
apt install btrfs-tools parted lvm2
 
# enable kernel module
 
modprobe btrfs
 
# format newly added disk, cfdisk is a graphical console fdisk
 
cfdisk /dev/sdb
 
# select gpt
# create [ New ] partition
# change [ Type ] to "LVM" -> we use lvm because this makes it easier to extend the disk later
# [ Write ] changes to disk and [ Quit ]
 
# introduce new partition schema to kernel
 
partprobe -s
ls -l /dev/ | grep sd
 
# now we should see sdb1
 
# create logical volume
pvcreate /dev/sdb1
vgcreate dedup_vg /dev/sdb1
 
# add volumes to the volumegroup (vg) (our first four Terybyte)
 
lvcreate -L +4T -n dedup_lv1 dedup_vg
 
pvs && vgs && lvs
 
# now you should see the volume group and the volumes
# we can now create btrfs filsystem over it
 
mkfs.btrfs /dev/dedup_vg/dedup_lv1
 
# and mount it
 
mkdir /mnt/dedup0
mount /dev/dedup_vg/dedup_lv1 /mnt/dedup0
 
# you can now check stats with
 
btrfs device stats /mnt/dedup0/
 
# to make this mount permanent edit /etc/fstab and add following line
/dev/dedup_vg/dedup_lv1 /mnt/dedup0/ btrfs defaults 0 0
 
# save and exit, and verify with
 
mount -va
 
 
 
# install bees (dedup agent):
# ---------------------------------
 
apt install git pkg-config build-essential btrfs-progs libbtrfs-dev uuid-dev markdown uuid-runtime
 
cd /root
 
git clone https://github.com/Zygo/bees.git
 
cd bees
 
make
 
make install
 
# find previously created uuid of btrfs device with "blkid" and copy it (Type=btrfs)
 
# enable beesd service
 
systemctl enable beesd@<uuid>.service
 
# since we need a config for our service
# create a new /etc/bees/<uuid>.conf
# and add following snippet, also edit parameters!
 
# -----------------------------------------------------------------
 
# Config for Bees: /etc/bees/beesd.conf.sample
## https://github.com/Zygo/bees
## It's a default values, change it, if needed
 
# How to use?
# Copy this file to a new file name and adjust the UUID below
 
# Which FS will be used
UUID=<uuid>
 
## System Vars
# Change carefully
WORK_DIR=/run/bees/
MNT_DIR="$WORK_DIR/mnt/$UUID"
BEESHOME="$MNT_DIR/.beeshome"
BEESSTATUS="$WORK_DIR/$UUID.status"
 
## Options to apply, see `beesd --help` for details
# OPTIONS="--strip-paths --no-timestamps"
 
## Bees DB size
# Hash Table Sizing
# sHash table entries are 16 bytes each
# (64-bit hash, 52-bit block number, and some metadata bits)
# Each entry represents a minimum of 4K on disk.
# unique data size    hash table size    average dedup block size
#     1TB                 4GB                  4K
#     1TB                 1GB                 16K
#     1TB               256MB                 64K
#     1TB                16MB               1024K
#    64TB                 1GB               1024K
#
# Size MUST be multiple of 128KB
# we use 8GB of hash space for 4TB of disk space, this also maps to memory, so you need 8GB of free heap!
# this is only a brief assumption on basis of a test drive we did with share data.
DB_SIZE=$((1024*1024*1024*8)) # 8G in bytes
 
#-----------------------------------------------------------------------------
 
# now start bee
 
systemctl start beesd@<uuid>
 
# check status with (expecially allocation problems, need more ram?)
 
systemctl systemctl status beesd@<uuid>
 
# or
 
journalctl -xe
 
journalctl -fu beesd@<uuid>