-
-
Save sorki/d81e20fac599961b09dc74f52c46fb72 to your computer and use it in GitHub Desktop.
wireguard, nats, containers
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { system ? builtins.currentSystem | |
| , config ? {} | |
| , pkgs ? import <nixpkgs> { inherit system config; } | |
| , lib ? pkgs.lib | |
| }: | |
| let | |
| inherit (import "${pkgs.path}/nixos/lib/testing-python.nix" { inherit system pkgs; }) makeTest; | |
| peer = (import ./lib/make-peer.nix) { inherit lib; }; | |
| wg-snakeoil-keys = import ./lib/wg-snakeoil-keys.nix; | |
| # XXX | |
| unstable = import /home/srk/git/nixpkgs/new-tests {}; | |
| in | |
| makeTest { | |
| name = "wireguard-nat"; | |
| meta = with pkgs.stdenv.lib.maintainers; { | |
| maintainers = [ sorki ]; | |
| }; | |
| nodes = { | |
| # wireguard server / router | |
| peer0 = peer { | |
| ip4 = "192.168.0.1"; | |
| ip6 = "fd00::1"; | |
| extraConfig = { | |
| networking.nat.enable = true; | |
| networking.nat.externalInterface = "eth1"; | |
| networking.nat.internalInterfaces = [ "wg0" ]; | |
| networking.firewall.allowedUDPPorts = [ 23542 ]; | |
| networking.wireguard.interfaces.wg0 = { | |
| ips = [ "10.23.42.1/32" "fc00::1/128" ]; | |
| listenPort = 23542; | |
| inherit (wg-snakeoil-keys.peer0) privateKey; | |
| /* | |
| postSetup = '' | |
| ${pkgs.iptables}/bin/iptables -t nat -A POSTROUTING -s 10.23.42.0/24 -j MASQUERADE | |
| ''; | |
| postShutdown = '' | |
| ${pkgs.iptables}/bin/iptables -t nat -D POSTROUTING -s 10.23.42.0/24 -j MASQUERADE | |
| ''; | |
| */ | |
| peers = [ | |
| { allowedIPs = [ "10.23.42.2/32" "fc00::2/128" ]; | |
| inherit (wg-snakeoil-keys.peer1) publicKey; | |
| } | |
| { allowedIPs = [ "10.23.42.3/32" "fc00::3/128" ]; | |
| inherit (wg-snakeoil-keys.peer2) publicKey; | |
| } | |
| { allowedIPs = [ "10.23.42.4/32" "fc00::4/128" ]; | |
| inherit (wg-snakeoil-keys.peer3) publicKey; | |
| } | |
| { allowedIPs = [ "10.23.42.5/32" "fc00::5/128" ]; | |
| inherit (wg-snakeoil-keys.peer4) publicKey; | |
| } | |
| { allowedIPs = [ "10.23.42.6/32" "fc00::6/128" ]; | |
| inherit (wg-snakeoil-keys.peer5) publicKey; | |
| } | |
| ]; | |
| }; | |
| }; | |
| }; | |
| # peer with nixos container | |
| # and internal ipv6 access | |
| peer1 = peer { | |
| ip6 = "fd00::2"; | |
| extraConfig = { | |
| networking.wireguard.interfaces.wg0 = { | |
| ips = [ "10.23.42.2/32" "fc00::2/128" ]; | |
| allowedIPsAsRoutes = true; | |
| inherit (wg-snakeoil-keys.peer1) privateKey; | |
| peers = lib.singleton { | |
| allowedIPs = [ "0.0.0.0/0" "::/0" ]; | |
| endpoint = "[fd00::1]:23542"; | |
| persistentKeepalive = 25; | |
| inherit (wg-snakeoil-keys.peer0) publicKey; | |
| }; | |
| }; | |
| # XXX | |
| environment.systemPackages = [ unstable.cntr ]; | |
| networking.nat.enable = true; | |
| networking.nat.enableIPv6 = true; | |
| networking.nat.externalInterface = "wg0"; | |
| networking.nat.internalInterfaces = [ "ve-+" ]; | |
| containers.test = { | |
| autoStart = true; | |
| privateNetwork = true; | |
| hostAddress = "172.16.0.1"; | |
| localAddress = "172.16.0.2"; | |
| hostAddress6 = "fa00::1"; | |
| localAddress6 = "fa00::2"; | |
| config = {}; | |
| }; | |
| }; | |
| }; | |
| # docker | |
| peer2 = peer { | |
| ip6 = "fd00::3"; | |
| extraConfig = { | |
| networking.wireguard.interfaces.wg0 = { | |
| ips = [ "10.23.42.3/32" "fc00::3/128" ]; | |
| allowedIPsAsRoutes = true; | |
| inherit (wg-snakeoil-keys.peer2) privateKey; | |
| peers = lib.singleton { | |
| allowedIPs = [ "0.0.0.0/0" "::/0" ]; | |
| endpoint = "[fd00::1]:23542"; | |
| persistentKeepalive = 25; | |
| inherit (wg-snakeoil-keys.peer0) publicKey; | |
| }; | |
| }; | |
| # XXX | |
| environment.systemPackages = [ unstable.cntr ]; | |
| networking.nat.enable = true; | |
| networking.nat.enableIPv6 = true; | |
| networking.nat.externalInterface = "wg0"; | |
| networking.nat.internalInterfaces = [ "docker+" ]; | |
| virtualisation.docker.extraOptions = "--ipv6 --fixed-cidr-v6=fa00:dac:1::/64 --iptables=false"; | |
| virtualisation.oci-containers = { | |
| backend = "docker"; | |
| containers.nginx = { | |
| image = "nginx-container"; | |
| imageFile = pkgs.dockerTools.examples.nginx; | |
| ports = ["8181:80"]; | |
| }; | |
| }; | |
| }; | |
| }; | |
| # some service, only tunnel v4/24 in allowed IPs | |
| peer3 = peer { | |
| ip4 = "192.168.0.4"; | |
| ip6 = "fd00::4"; | |
| extraConfig = { | |
| networking.wireguard.interfaces.wg0 = { | |
| ips = [ "10.23.42.4/32" "fc00::4/128" ]; | |
| allowedIPsAsRoutes = true; | |
| inherit (wg-snakeoil-keys.peer3) privateKey; | |
| peers = lib.singleton { | |
| allowedIPs = [ "10.23.42.0/24" ]; | |
| endpoint = "[fd00::1]:23542"; | |
| persistentKeepalive = 25; | |
| inherit (wg-snakeoil-keys.peer0) publicKey; | |
| }; | |
| }; | |
| }; | |
| }; | |
| # some service, everything thru tunnel | |
| peer4 = peer { | |
| ip4 = "192.168.0.5"; | |
| ip6 = "fd00::5"; | |
| extraConfig = { | |
| networking.wireguard.interfaces.wg0 = { | |
| ips = [ "10.23.42.5/32" "fc00::5/128" ]; | |
| allowedIPsAsRoutes = true; | |
| inherit (wg-snakeoil-keys.peer4) privateKey; | |
| peers = lib.singleton { | |
| allowedIPs = [ "0.0.0.0/0" "::/0" ]; | |
| endpoint = "[fd00::1]:23542"; | |
| persistentKeepalive = 25; | |
| inherit (wg-snakeoil-keys.peer0) publicKey; | |
| }; | |
| }; | |
| }; | |
| }; | |
| # podman | |
| # no wan v4, all v4 thru tunnel | |
| # only lan v6 thru tunnel | |
| peer5 = peer { | |
| ip6 = "fd00::6"; | |
| extraConfig = { | |
| networking.wireguard.interfaces.wg0 = { | |
| ips = [ "10.23.42.6/32" "fc00::6/128" ]; | |
| allowedIPsAsRoutes = true; | |
| inherit (wg-snakeoil-keys.peer5) privateKey; | |
| peers = lib.singleton { | |
| allowedIPs = [ "0.0.0.0/0" "fc00::/64" ]; | |
| endpoint = "[fd00::1]:23542"; | |
| persistentKeepalive = 25; | |
| inherit (wg-snakeoil-keys.peer0) publicKey; | |
| }; | |
| }; | |
| # XXX | |
| environment.systemPackages = [ unstable.cntr ]; | |
| environment.etc."cni/net.d/10-podman-ipv6-bridge.conflist".text = builtins.readFile ./10-podman-ipv6-bridge.conflist; | |
| virtualisation.oci-containers = { | |
| backend = "podman"; | |
| containers.nginx = { | |
| image = "nginx-container"; | |
| imageFile = pkgs.dockerTools.examples.nginx; | |
| ports = ["8181:80"]; | |
| extraOptions = [ "--network=podman-ipv6" ]; | |
| }; | |
| }; | |
| }; | |
| }; | |
| # only part of hosts network, no wireguard peering | |
| external = peer { | |
| ip4 = "192.168.0.10"; | |
| ip6 = "fd00::10"; | |
| extraConfig = { }; | |
| }; | |
| }; | |
| testScript = '' | |
| start_all() | |
| peer0.wait_for_unit("wireguard-wg0.service") | |
| peer1.wait_for_unit("wireguard-wg0.service") | |
| peer2.wait_for_unit("wireguard-wg0.service") | |
| peer3.wait_for_unit("wireguard-wg0.service") | |
| peer4.wait_for_unit("wireguard-wg0.service") | |
| peer5.wait_for_unit("wireguard-wg0.service") | |
| def inspectPeer(num): | |
| p = getattr(sys.modules[__name__], "peer{}".format(num)) | |
| print("Peer {}".format(num)) | |
| print(p.succeed("ip a")) | |
| print(p.succeed("ip r")) | |
| print(p.succeed("ip -6 a")) | |
| print(p.succeed("ip -6 r")) | |
| print(p.succeed("wg")) | |
| inspectPeer(0) | |
| peer1.succeed("ping -c1 fc00::2") | |
| peer1.succeed("ping -c1 fc00::3") | |
| peer1.fail("ping -c1 fc00::4") | |
| peer1.succeed("ping -c1 fc00::5") | |
| peer1.succeed("ping -c1 fc00::6") | |
| inspectPeer(1) | |
| # wg v6 allowed | |
| peer1.succeed("ping -c1 fc00::1") | |
| peer1.succeed("ping -c1 fd00::1") | |
| peer1.succeed("ping -c1 10.23.42.1") | |
| # we can reach external via wg | |
| peer1.succeed("ping -c1 192.168.0.10") | |
| print(peer1.succeed("ls -l $( which cntr )")) | |
| peer1.wait_for_unit("container@test.service") | |
| peer1.succeed("cntr attach test sh -- -c 'ping -c1 10.23.42.1'") | |
| peer1.succeed("cntr attach test sh -- -c 'ping -c1 192.168.0.10'") | |
| peer1.succeed("cntr attach test sh -- -c 'ping -c1 fc00::1'") | |
| # our localhost | |
| peer1.succeed("cntr attach test sh -- -c 'ping -c1 fd00::2'") | |
| # cannot ping this one as there's a more specific route out of eth1 | |
| # but why it works from host?! | |
| # proly because NAT has external wg and route points to eth1 | |
| peer1.fail("cntr attach test sh -- -c 'ping -c1 fd00::1'") | |
| inspectPeer(2) | |
| peer2.succeed("ping -c1 fc00::1") | |
| peer2.succeed("ping -c1 fd00::1") | |
| peer2.succeed("ping -c1 10.23.42.1") | |
| peer2.succeed("ping -c1 fd00::2") | |
| peer2.succeed("ping -c1 10.23.42.2") | |
| peer2.wait_for_unit("docker-nginx.service") | |
| peer2.sleep(1) | |
| peer2.succeed("cntr attach -t docker nginx sh -- -c 'ping -c1 192.168.0.10'") | |
| peer2.succeed("cntr attach -t docker nginx sh -- -c 'ping -c1 fc00::1'") | |
| inspectPeer(3) | |
| peer3.succeed("ping -c1 fd00::1") | |
| peer3.succeed("ping -c1 10.23.42.1") | |
| peer3.succeed("ping -c1 10.23.42.2") | |
| peer3.succeed("ping -c1 10.23.42.3") | |
| inspectPeer(4) | |
| peer4.succeed("ping -c1 fc00::1") | |
| peer4.succeed("ping -c1 fd00::1") | |
| peer4.succeed("ping -c1 10.23.42.1") | |
| peer4.succeed("ping -c1 10.23.42.2") | |
| peer4.succeed("ping -c1 10.23.42.3") | |
| inspectPeer(5) | |
| peer5.succeed("ping -c1 fc00::1") | |
| peer5.succeed("ping -c1 fc00::2") | |
| peer5.succeed("ping -c1 fc00::3") | |
| peer5.succeed("ping -c1 fd00::1") | |
| peer5.succeed("ping -c1 10.23.42.1") | |
| peer5.succeed("ping -c1 10.23.42.2") | |
| peer5.succeed("ping -c1 10.23.42.3") | |
| peer5.wait_for_unit("podman-nginx.service") | |
| peer5.sleep(1) | |
| peer5.succeed("cntr attach -t podman nginx sh -- -c 'ping -c1 192.168.0.10'") | |
| peer5.succeed("cntr attach -t podman nginx sh -- -c 'ping -c1 fc00::1'") | |
| peer5.succeed("cntr attach -t podman nginx sh -- -c 'ping -c1 fd00::1'") | |
| peer5.succeed("cntr attach -t podman nginx sh -- -c 'ping -c1 fd00::10'") | |
| ''; | |
| } | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment