script to create many shared blobs

many-shared-blobs.sh

#!/bin/bash
MON=1 OSD=1 MDS=0 MGR=1 RGW=0 NFS=0 ../src/vstart.sh -l -n --without-dashboard
./bin/ceph osd pool create test_pool 64 --pg_num_min 64


export FIO=${FIO:-$(which fio)}
export TEST_SPACE="--size=60000M --nrfiles=15000"
export DISTR=pareto:0.30:0 #this gives 30% objects handle 70% requests
export TOTAL_TRANSFER=15000M
export IOSIZE="--bssplit=4k/16:8k/10:12k/9:16k/8:20k/7:24k/7:28k/6:32k/6:36k/5:40k/5:44k/4:48k/4:52k/4:56k/3:60k/3:64k/3"

FIO_ARGS="--ioengine=rados --pool=test_pool --invalidate=0 \
	--touch_objects=0 \
	--output-format=json,normal \
	--group_reporting \
	--randrepeat=0 \
	${TEST_SPACE} \
	--filename_format=object.\$filenum"

CEPH_CONF=$(pwd)/ceph.conf LD_LIBRARY_PATH=$(pwd)/lib ${FIO} ${FIO_ARGS} \
    --name=FILL --iodepth=64 --rw=write --bs=4M


../src/stop.sh
../src/vstart.sh -l -k
echo CEPH_CONF=$(pwd)/ceph.conf LD_LIBRARY_PATH=$(pwd)/lib ${FIO} ${FIO_ARGS} \
    --name=frag-1 \
    --file_service_type=${DISTR} -norandommap \
    ${IOSIZE} --iodepth=64 --io_size=${TOTAL_TRANSFER} \
    --rw=randwrite

export TOTAL_TRANSFER=2000M
export IOSIZE="--bssplit=4k/100"

for ((s=0;s<10;s++)) 
do 
./bin/rados -p test_pool mksnap snap-${s}
CEPH_CONF=$(pwd)/ceph.conf LD_LIBRARY_PATH=$(pwd)/lib ${FIO} ${FIO_ARGS} \
    --name=snap-${s} \
    --file_service_type=${DISTR} -norandommap \
    ${IOSIZE} --iodepth=64 --io_size=${TOTAL_TRANSFER} \
    --rw=randwrite
done

This script creates ~3.7M of shared blobs, uses ~80GB of data.
Dataset created this way causes fsck to consume at the peak ~5.3GB ram.
To create more blobs one should modify
export TEST_SPACE="--size=60000M --nrfiles=15000"
Each object is 4MB.