README.md

NOTE: /hugetlbsfs is expected to be mounted NOTE: Control numa core assignment by setting baseCore. First VM will get baseCore+1 assigned, second VM baseCore+2, etc

Building resources

$ nix-build --arg baseCore 0 --arg numVms 12 -A test_env vms-domen.nix -o test_env
$ cp -L test_env/* /mnt/mydisk/

Running VMs

First make sure all snabb processes are running and have sockets in current directory names like $i.socket (from 1.socket to 12.socket).

NixOS

$ nix-shell -A run vms-domen.nix
$ sudo ./test_env/run

Other distros

Make sure current directory is writeable by root and that build artifacts are copied to ./test_env

Install qemu, tmux and utillinux. Now run:

$ sudo ./test_env/run

vms-domen.nix

{ nixpkgs ? (fetchTarball https://github.com/NixOS/nixpkgs/archive/632100bdf4588f27a886ec5df38c1d4a9e136d5d.tar.gz)
, numVms ? 10
, baseCore ? 0
}:

with (import nixpkgs {
    config.packageOverrides = pkgs:
  {
         qemu = pkgs.qemu.overrideDerivation (super: {
           name = "qemu-2.4.1-snabb";
           src = pkgs.fetchurl {
             url = "http://wiki.qemu-project.org/download/qemu-2.4.1.tar.bz2";
             sha256 = "0xx1wc7lj5m3r2ab7f0axlfknszvbd8rlclpqz4jk48zid6czmg3";
           };
           patches = [ (pkgs.lib.head super.patches) (pkgs.fetchurl {
             url = "https://github.com/SnabbCo/qemu/commit/f393aea2301734647fdf470724433f44702e3fb9.patch";
             sha256 = "0hpnfdk96rrdaaf6qr4m4pgv40dw7r53mg95f22axj7nsyr8d72x";
           })];
         });
  
  };
});
with lib;
with vmTools;


let
  # modules and NixOS config for plain qemu image
  modules = [
    "${nixpkgs}/nixos/modules/profiles/qemu-guest.nix"
    ({config, pkgs, ...}: {
      environment.systemPackages = with pkgs; [ inetutils screen python pciutils ethtool tcpdump netcat iperf iptables ];
      fileSystems."/".device = "/dev/vda1";
      boot.loader.grub.device = "/dev/vda";

      # settings needed by tests
      networking.firewall.enable = mkOverride 150 false;
      services.mingetty.autologinUser = "root";
      users.extraUsers.root.initialHashedPassword = mkOverride 150 "";
      networking.usePredictableInterfaceNames = false;
    })
    ({config, pkgs, lib, ...}:
      let
        dpdk_bind = pkgs.fetchurl {
          url = "https://raw.githubusercontent.com/scylladb/dpdk/8ea56fadc9a49c575bee6bb3892bc17dd9ec4ab6/tools/dpdk_nic_bind.py";
          sha256 = "0z8big9gh49q9kh0jjg1p9g5ywwvb130r3bmhhpbgx7blhk9zb7f";
        };
      in {
        systemd.services.dpdk = {
          wantedBy = [ "multi-user.target" ];
          after = [ "network.target" ];
          path = with pkgs; [ kmod python pciutils iproute utillinux ];
          script = ''
            mkdir -p /hugetlbfs
            mount -t hugetlbfs nodev /hugetlbfs
            echo 64 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
            MODULE_DIR=/run/current-system/kernel-modules/lib/modules modprobe uio
            insmod ${config.boot.kernelPackages.dpdk.kmod}/lib/modules/${config.boot.kernelPackages.kernel.modDirVersion}/kernel/drivers/net/igb_uio.ko
            #insmod ${config.boot.kernelPackages.dpdk}/kmod/igb_uio.ko
            python ${dpdk_bind} --bind=igb_uio 00:03.0
            ${config.boot.kernelPackages.dpdk.examples}/bin/l2fwd -c 0x1 -n1 -- -p 0x1
          '';
        };
      }
    )
  ];
  config = (import "${nixpkgs}/nixos/lib/eval-config.nix" { inherit modules; }).config;
in rec {
  # files needed for some tests
  test_env = runCommand "test_env" (rec {
      passthru.config = config;
      qemu_img = lib.makeOverridable (import "${nixpkgs}/nixos/lib/make-disk-image.nix") {
        name = "snabb-nixos-minimal";
        inherit lib config pkgs;
        diskSize = 2 * 1024;
        format = "qcow2";
      };
    }) ''
    mkdir -p $out
    ln -s $qemu_img/nixos.qcow2 $out/qemu.img
    ln -s ${config.system.build.toplevel}/kernel $out/bzImage
    ln -s ${config.system.build.toplevel}/initrd $out/initrd
    ln -s ${config.system.build.toplevel}/init $out/init
    ln -s ${script} $out/run
  '';
  run = runCommand "run" {
    buildInputs = [ tmux qemu utillinux ];
  } "";
  script = writeScript "script.sh" ''
    tmux set remain-on-exit on
    tmux kill-session -t vms || true
    tmux new-session -d -s "vms" 
    for i in {1..${toString numVms}}; do
      qemu-img create -b ./test_env/qemu.img -f qcow2 qemu$i.img
      tmux new-window -a -d -n "qemu$i" -t vms "taskset -c $[${toString baseCore} + $i] qemu-system-x86_64 \
        -initrd ./test_env/initrd \
        -kernel ./test_env/bzImage \
        -append 'earlyprintk root=/dev/vda init=$(readlink -f ./test_env/init) rw console=ttyS0 ip=fe80::5054:ff:fe00:0000' \
        -m 512 -numa node,memdev=mem -object memory-backend-file,id=mem,size=512M,mem-path=/hugetlbfs,share=on \
        -netdev type=vhost-user,id=net0,chardev=char0 -chardev socket,id=char0,path=$i.socket,server \
        -device virtio-net-pci,netdev=net0,mac=00:00:00:00:00:$(printf '%02d' $i),mq=off,vectors=3 \
        -M pc -smp 1 -cpu host --enable-kvm \
        -drive if=virtio,file=./qemu$i.img \
        -nographic"
    done
    echo "Attach to see tmux vms output:"
    echo "  $ sudo tmux a -t vms"
    echo "To kill vms:"
    echo "  $ sudo tmux kill-session -t vms"
  '';
}