A Vagrantfile illustrating bad ZFS performance with primarycache=metadata

Vagrantfile

Vagrant.configure("2") do |config|
  config.vm.box = "ubuntu/xenial64"

  # disable serial port to increase boot speed
  config.vm.provider "virtualbox" do |v|
    v.customize [ "modifyvm", :id, "--uartmode1", "disconnected" ]
  end

  config.vm.provision "shell", inline: <<-SHELL
    apt-get update
    apt-get install -y zfs
    echo ======================================
    echo Creating backing file...
    truncate -s 5G testpool.bin
    echo Creating zpool...
    zpool create -O primarycache=metadata testpool $PWD/testpool.bin
    echo Setting xattr=sa and atime=off to eliminate that as an issue...
    zfs set xattr=sa testpool
    zfs set atime=off testpool
    echo Creating test file...
    dd if=/dev/zero of=/testpool/testfile bs=1K count=1M # 1GB test file
    echo Initialization finished.
    echo ======================================
    echo Starting tests!
    echo Testing read with primarycache=metadata...
    dd if=/testpool/testfile of=/dev/null count=50K
    echo Setting primarycache=all
    zfs set primarycache=all testpool
    echo Testing read again...
    dd if=/testpool/testfile of=/dev/null count=50K
    echo Setting primarycache back to metadata.
    zfs set primarycache=metadata testpool
    echo Testing read again...
    dd if=/testpool/testfile of=/dev/null count=50K
    echo Finished! Use \\'vagrant ssh\\' to inspect.
  SHELL
end