- verify machine has extra disks. e.g.
$ lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 893.8G 0 disk
__sda1 8:1 0 893.8G 0 part /
nvme0n1 259:0 0 1.5T 0 disk
nvme3n1 259:1 0 1.5T 0 disk
nvme1n1 259:2 0 1.5T 0 disk
nvme5n1 259:3 0 1.5T 0 disk
nvme6n1 259:4 0 1.5T 0 disk
nvme2n1 259:5 0 1.5T 0 disk
nvme4n1 259:6 0 1.5T 0 disk
nvme7n1 259:7 0 1.5T 0 disk
- start cluster:
sudo MON=1 OSD=1 MDS=0 MGR=0 RGW=1 ../src/vstart.sh -n -X --nolockdep --without-dashboard \
--bluestore-devs /dev/nvme7n1 -o bluestore_block_db_path=/dev/nvme6n1 -o bluestore_block_wal_path=/dev/nvme5n1 \
-o rgw_dynamic_resharding=false -o osd_pool_default_pg_num=128 -o osd_pool_default_pgp_num=128 \
-o mon_max_pg_per_osd=32768 -o mon_pg_warn_max_per_osd=32768 -o osd_pool_default_pg_autoscale_mode=warn
- use hsbench to load the system:
hsbench -a 0555b35654ad1656d804 -s h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== \
-u http://localhost:8000 -bp bk -m ipd -t 64 -z 1K -l 5 -d 120 -b 6
note to use small objects (1K in the above example), so that the notification overhead is relatively high
- hsbench will output the average IO/s
- make sure that a kafka broker run on the host
- create a persistent topic that points to the broker and a notification on the bucket created by hsbench
aws --region=default --endpoint-url http://localhost:8000 sns create-topic --name=fishtopic0 \
--attributes='{"push-endpoint": "kafka://localhost", "persistent": "true"}'
aws --region=default --endpoint-url http://localhost:8000 sns create-topic --name=fishtopic1 \
--attributes='{"push-endpoint": "kafka://localhost", "persistent": "true"}'
aws --region=default --endpoint-url http://localhost:8000 sns create-topic --name=fishtopic2 \
--attributes='{"push-endpoint": "kafka://localhost", "persistent": "true"}'
aws --region=default --endpoint-url http://localhost:8000 sns create-topic --name=fishtopic3 \
--attributes='{"push-endpoint": "kafka://localhost", "persistent": "true"}'
aws --region=default --endpoint-url http://localhost:8000 sns create-topic --name=fishtopic4 \
--attributes='{"push-endpoint": "kafka://localhost", "persistent": "true"}'
aws --region=default --endpoint-url http://localhost:8000 sns create-topic --name=fishtopic5 \
--attributes='{"push-endpoint": "kafka://localhost", "persistent": "true"}'
aws --region=default --endpoint-url http://localhost:8000 s3api put-bucket-notification-configuration \
--bucket bk000000000000 --notification-configuration='{"TopicConfigurations": [{"Id": "notif1", "TopicArn": "arn:aws:sns:default::fishtopic0", "Events": []}]}'
aws --region=default --endpoint-url http://localhost:8000 s3api put-bucket-notification-configuration \
--bucket bk000000000001 --notification-configuration='{"TopicConfigurations": [{"Id": "notif1", "TopicArn": "arn:aws:sns:default::fishtopic1", "Events": []}]}'
aws --region=default --endpoint-url http://localhost:8000 s3api put-bucket-notification-configuration \
--bucket bk000000000002 --notification-configuration='{"TopicConfigurations": [{"Id": "notif1", "TopicArn": "arn:aws:sns:default::fishtopic2", "Events": []}]}'
aws --region=default --endpoint-url http://localhost:8000 s3api put-bucket-notification-configuration \
--bucket bk000000000003 --notification-configuration='{"TopicConfigurations": [{"Id": "notif1", "TopicArn": "arn:aws:sns:default::fishtopic3", "Events": []}]}'
aws --region=default --endpoint-url http://localhost:8000 s3api put-bucket-notification-configuration \
--bucket bk000000000004 --notification-configuration='{"TopicConfigurations": [{"Id": "notif1", "TopicArn": "arn:aws:sns:default::fishtopic4", "Events": []}]}'
aws --region=default --endpoint-url http://localhost:8000 s3api put-bucket-notification-configuration \
--bucket bk000000000005 --notification-configuration='{"TopicConfigurations": [{"Id": "notif1", "TopicArn": "arn:aws:sns:default::fishtopic5", "Events": []}]}'
aws --region=default --endpoint-url http://localhost:8000 s3api put-bucket-notification-configuration \
--bucket bk000000000000 --notification-configuration='{"TopicConfigurations": [{"Id": "notif1", "TopicArn": "arn:aws:sns:default::fishtopic", "Events": []}]}'
- check the change in IO/s and CPU