test.nix

# This replicates, as closely as possible:
#
# - INTERNET a machine which resembles the public internet
# - ROUTER a machine connecting BACKEND and RUNNER to INTERNET
# - BACKEND a physical machine behind ROUTER
# - RUNNER a physical machine behind ROUTER
# - Inside RUNNER, a NixOS Container called MALICIOUS
#
# The goal is to produce a standard network access scenario of:
#
# - BACKEND, RUNNER, and MALICIOUS can talk to INTERNET through ROUTER
# - BACKEND and RUNNER can talk to each other
# - MALICIOUS can *only* talk to:
#    - INTERNET over TCP port 80
#    - RUNNER over UDP port 53 (DNS)
#
# MALICIOUS should not be able to talk to any other services on:
# - BACKEND
# - RUNNER
# - ROUTER
#
####
#
# This test should be run twice:
#
# 1) verifying the configuration works, insecurely allowing all traffic
# 2) with firewall hardening, restricting traffic
#
# This first and second test is intended to ensure the mitigations in
# place are effective, and not simply a network misconfiguration.
#
####
#
# Note when setting up VLANs:
# when virtualisation.vlans [ vlanA vlanB vlanC ]
# the IP for vlanA must be at eth1, vlanB at eth2 vlanC at eth3
# due to the position in the vlans list.
#
let
  secure = true;

  vlans = rec {
    internet = {
      id = 2;
      prefix = 24;
      cidr = "1.1.1.0/24";
      target-ip = "1.1.1.1";
      router-ip = "1.1.1.100";
    };
    internal-192-168 = {
      id = 3;
      prefix = 24;
      cidr = "192.168.100.0/24";
      router-ip = "192.168.100.3";
      backend-ip = "192.168.100.111";
      runner-ip = "192.168.100.222";
    };
    internal-172-16 = {
      id = 4;
      prefix = 12;
      cidr = "172.16.0.0/12";
      router-ip = "172.16.100.3";
      backend-ip = "172.16.100.111";
      runner-ip = "172.16.100.222";
    };
    internal-10 = {
      id = 5;
      prefix = 8;
      cidr = "10.0.0.0/8";
      router-ip = "10.1.100.3";
      backend-ip = "10.1.100.111";
      runner-ip = "10.1.100.222";
    };
  };
in import <nixpkgs/nixos/tests/make-test.nix> ({ pkgs, ...} : {
  name = "config-isolation";

  nodes = let
    inherit (pkgs.lib) mkOverride;
  in {
    internet = {
      services.nginx = {
        enable = true;
        virtualHosts.localhost = {
          listen = [
            { addr = "0.0.0.0"; port = 80; }
            { addr = "0.0.0.0"; port = 81; }
          ];
          root = pkgs.writeTextDir "index.html" "hello!";
        };
      };

      networking = {
        firewall.allowedTCPPorts = [ 80 81 ];
        dhcpcd.enable = false;
        interfaces.eth1.ipv4.addresses = mkOverride 0 [
          { address = vlans.internet.target-ip; prefixLength = vlans.internet.prefix; }
        ];
      };

      virtualisation.vlans = [ vlans.internet.id ];
    };

    router = { config, ... }: {
      boot.kernel.sysctl = {
        "net.ipv4.conf.eth2.forwarding" = 1;
        "net.ipv4.conf.eth3.forwarding" = 1;
        "net.ipv4.conf.eth4.forwarding" = 1;
      };
      networking = {
        nat = {
          enable = true;
          externalInterface = "eth1";
          internalInterfaces = [
            "eth2"
            "eth3"
            "eth4"
          ];
          internalIPs = [
            vlans.internal-192-168.cidr
            vlans.internal-172-16.cidr
            vlans.internal-10.cidr
          ];
        };
        firewall = {
          allowedTCPPorts = [ 80 ];
          extraCommands = ''
            ip46tables -I FORWARD -m state NEW -i eth2 -o eth1 -j ACCEPT
            ip46tables -I FORWARD -m state NEW -i eth3 -o eth1 -j ACCEPT
            ip46tables -I FORWARD -m state NEW -i eth4 -o eth1 -j ACCEPT
            ip46tables -I FORWARD -m state --state ESTABLISHED,RELATED -j ACCEPT
            ip46tables -A FORWARD -i eth1 -j DROP
          '';
        };
        dhcpcd.enable = false;
        interfaces.eth1.ipv4.addresses = mkOverride 0 [
          { address = vlans.internet.router-ip; prefixLength = vlans.internet.prefix; }
        ];
        interfaces.eth2.ipv4.addresses = mkOverride 0 [
          { address = vlans.internal-192-168.router-ip; prefixLength = vlans.internal-192-168.prefix; }
        ];
        interfaces.eth3.ipv4.addresses = mkOverride 0 [
          { address = vlans.internal-172-16.router-ip; prefixLength = vlans.internal-172-16.prefix; }
        ];
        interfaces.eth4.ipv4.addresses = mkOverride 0 [
          { address = vlans.internal-10.router-ip; prefixLength = vlans.internal-10.prefix; }
        ];
      };

      virtualisation.vlans = [
        vlans.internet.id
        vlans.internal-192-168.id
        vlans.internal-172-16.id
        vlans.internal-10.id
      ];

      services.nginx = {
        enable = true;
        virtualHosts.localhost = {
          listen = [
            { addr = "0.0.0.0"; port = 80; }
          ];
          root = pkgs.writeTextDir "index.html" "DANGER";
        };
      };
    };

    backend = {
      services.nginx = {
        enable = true;
        virtualHosts.localhost = {
          listen = [
            { addr = "0.0.0.0"; port = 80; }
          ];
          root = pkgs.writeTextDir "index.html" "DANGER";
        };
      };

      networking = {
        firewall.allowedTCPPorts = [ 80 ];
        dhcpcd.enable = false;
        defaultGateway = vlans.internal-192-168.router-ip;
        interfaces.eth1.ipv4.addresses = mkOverride 0 [
          { address = vlans.internal-192-168.backend-ip; prefixLength = vlans.internal-192-168.prefix; }
        ];
        interfaces.eth2.ipv4.addresses = mkOverride 0 [
          { address = vlans.internal-172-16.backend-ip; prefixLength = vlans.internal-172-16.prefix; }
        ];
        interfaces.eth3.ipv4.addresses = mkOverride 0 [
          { address = vlans.internal-10.backend-ip; prefixLength = vlans.internal-10.prefix; }
        ];
      };

      virtualisation.vlans = [
        vlans.internal-192-168.id
        vlans.internal-172-16.id
        vlans.internal-10.id
      ];
    };

    runner = {
      boot.kernel.sysctl = {
        "net.ipv4.conf.all.forwarding" = 1;
      };

      services.nginx = {
        enable = true;
        virtualHosts.localhost = {
          listen = [
            { addr = "0.0.0.0"; port = 80; }
          ];
          root = pkgs.writeTextDir "index.html" "DANGER";
        };
      };

      networking = {
        firewall = {
          allowedTCPPorts = [ 80 ];
          extraCommands = if secure then ''
            iptables -A FORWARD -i "br0" --dest 192.168.0.0/16 -j DROP
            iptables -A FORWARD -i "br0" --dest 172.16.0.0/12 -j DROP
            iptables -A FORWARD -i "br0" --dest 10.0.0.0/8 -j DROP

            iptables -I nixos-fw -i "br0" --dest 192.168.0.0/16 -j DROP
            iptables -I nixos-fw -i "br0" --dest 172.16.0.0/12 -j DROP
            iptables -I nixos-fw -i "br0" --dest 10.0.0.0/8 -j DROP

            ip46tables -A FORWARD -i "br0" -o "eth1" -p tcp --dport 80 -j ACCEPT
            ip46tables -A FORWARD -i "br0" -j DROP
          '' else "";
        };
        dhcpcd.enable = false;
        defaultGateway = vlans.internal-192-168.router-ip;
        interfaces.eth1.ipv4.addresses = mkOverride 0 [
          { address = vlans.internal-192-168.runner-ip; prefixLength = vlans.internal-192-168.prefix; }
        ];
        interfaces.eth2.ipv4.addresses = mkOverride 0 [
          { address = vlans.internal-172-16.runner-ip; prefixLength = vlans.internal-172-16.prefix; }
        ];
        interfaces.eth3.ipv4.addresses = mkOverride 0 [
          { address = vlans.internal-10.runner-ip; prefixLength = vlans.internal-10.prefix; }
        ];

        interfaces.br0.ipv4.addresses = mkOverride 0 [
          { address = "192.168.200.1"; prefixLength = 24; }
        ];

        nat = {
          enable = true;
          externalInterface = "eth1";
          internalInterfaces = [ "br0" ];
          internalIPs = [
            "192.168.200.0/24"
          ];
        };

        bridges.br0.interfaces = [];
      };

      virtualisation.vlans = [
        vlans.internal-192-168.id
        vlans.internal-172-16.id
        vlans.internal-10.id
      ];

      containers.malicious = {
        autoStart = true;
        privateNetwork = true;
        hostBridge = "br0";
        localAddress = "192.168.200.2/24";
        config = {
          networking.defaultGateway = "192.168.200.1";
        };
      };
    };
  };

  testScript = let
    commands = [
      "nixos-container run malicious -- curl --connect-timeout 1 http://${vlans.internet.target-ip}:81"
      # try talking to router
      "nixos-container run malicious -- curl --connect-timeout 1 http://${vlans.internal-192-168.router-ip}:80"
      # TODO: uncomment these by making 192.168.200.0/24 talk to 172.16.0.0/12 and 10.0.0.0/8 over router
      #"nixos-container run malicious -- curl --connect-timeout 1 http://${vlans.internal-172-16.router-ip}:80"
      #"nixos-container run malicious -- curl --connect-timeout 1 http://${vlans.internal-10.router-ip}:80"

      # try talking to backend
      "nixos-container run malicious -- curl --connect-timeout 1 http://${vlans.internal-192-168.backend-ip}:80"
      # TODO: uncomment these by making 192.168.200.0/24 talk to 172.16.0.0/12 and 10.0.0.0/8 over router
      #"nixos-container run malicious -- curl --connect-timeout 1 http://${vlans.internal-172-16.backend-ip}:80"
      #"nixos-container run malicious -- curl --connect-timeout 1 http://${vlans.internal-10.backend-ip}:80"

      # try talking to runner (host)
      "nixos-container run malicious -- curl --connect-timeout 1 http://${vlans.internal-192-168.runner-ip}:80"
      # TODO: uncomment these by making 192.168.200.0/24 talk to 172.16.0.0/12 and 10.0.0.0/8 over router
      #"nixos-container run malicious -- curl --connect-timeout 1 http://${vlans.internal-172-16.runner-ip}:80"
      #"nixos-container run malicious -- curl --connect-timeout 1 http://${vlans.internal-10.runner-ip}:80"

      # try talking to runner over the bridge's IP
      "nixos-container run malicious -- curl --connect-timeout 1 http://${"192.168.200.1"}:80"
    ];

    expectFn = if secure then "fail" else "succeed";

    expect = builtins.concatStringsSep "\n"
      (builtins.map (
        cmd: "$runner->${expectFn}(\"${cmd}\");"
      ) commands);
  in ''
    $internet->start;
    $router->start;
    $backend->start;
    $runner->start;

    $router->waitForUnit("multi-user.target");
    $router->succeed("systemd-cat ip route");
    $router->succeed("systemd-cat curl http://${vlans.internal-192-168.router-ip}:80");
    $router->succeed("systemd-cat curl http://${vlans.internal-172-16.router-ip}:80");
    $router->succeed("systemd-cat curl http://${vlans.internal-10.router-ip}:80");

    $internet->waitForUnit("multi-user.target");
    $router->succeed("systemd-cat ip route");
    $router->succeed("systemd-cat curl http://${vlans.internet.target-ip}:80");
    $router->succeed("systemd-cat curl http://${vlans.internet.target-ip}:81");
    $internet->succeed("systemd-cat curl --connect-timeout 2 http://${vlans.internet.router-ip}:80");

    $backend->waitForUnit("multi-user.target");
    $backend->succeed("systemd-cat ip route");
    # try talking to itself
    $backend->succeed("systemd-cat curl http://${vlans.internal-192-168.backend-ip}:80");
    $backend->succeed("systemd-cat curl http://${vlans.internal-172-16.backend-ip}:80");
    $backend->succeed("systemd-cat curl http://${vlans.internal-10.backend-ip}:80");
    # try talking to the router
    $backend->succeed("systemd-cat curl http://${vlans.internal-192-168.router-ip}:80");
    $backend->succeed("systemd-cat curl http://${vlans.internal-172-16.router-ip}:80");
    $backend->succeed("systemd-cat curl http://${vlans.internal-10.router-ip}:80");
    # try talking to the internet
    $backend->succeed("systemd-cat curl http://${vlans.internet.target-ip}:80");
    $backend->succeed("systemd-cat curl http://${vlans.internet.target-ip}:81");
    # try having router talk to backend
    $router->succeed("systemd-cat curl http://${vlans.internal-192-168.backend-ip}:80");
    $router->succeed("systemd-cat curl http://${vlans.internal-172-16.backend-ip}:80");
    $router->succeed("systemd-cat curl http://${vlans.internal-10.backend-ip}:80");

    $runner->waitForUnit("multi-user.target");
    $runner->succeed("systemd-cat ip route");
    # try talking to itself
    $runner->succeed("systemd-cat curl http://${vlans.internal-192-168.runner-ip}:80");
    $runner->succeed("systemd-cat curl http://${vlans.internal-172-16.runner-ip}:80");
    $runner->succeed("systemd-cat curl http://${vlans.internal-10.runner-ip}:80");
    # try talking to the router
    $runner->succeed("systemd-cat curl http://${vlans.internal-192-168.router-ip}:80");
    $runner->succeed("systemd-cat curl http://${vlans.internal-172-16.router-ip}:80");
    $runner->succeed("systemd-cat curl http://${vlans.internal-10.router-ip}:80");
    # try talking to the backend
    $runner->succeed("systemd-cat curl http://${vlans.internal-192-168.backend-ip}:80");
    $runner->succeed("systemd-cat curl http://${vlans.internal-172-16.backend-ip}:80");
    $runner->succeed("systemd-cat curl http://${vlans.internal-10.backend-ip}:80");
    # try talking to the internet
    $runner->succeed("systemd-cat curl http://${vlans.internet.target-ip}:80");
    $runner->succeed("systemd-cat curl http://${vlans.internet.target-ip}:81");
    # try having backend talk to runner
    $backend->succeed("systemd-cat curl http://${vlans.internal-192-168.runner-ip}:80");
    $backend->succeed("systemd-cat curl http://${vlans.internal-172-16.runner-ip}:80");
    $backend->succeed("systemd-cat curl http://${vlans.internal-10.runner-ip}:80");
    # try having router talk to runner
    $router->succeed("systemd-cat curl http://${vlans.internal-192-168.runner-ip}:80");
    $router->succeed("systemd-cat curl http://${vlans.internal-172-16.runner-ip}:80");
    $router->succeed("systemd-cat curl http://${vlans.internal-10.runner-ip}:80");

    $runner->succeed("nixos-container run malicious -- curl http://${vlans.internet.target-ip}:80");
    $runner->succeed("systemd-cat iptables -L");
    ${expect}
  '';