Skip to content

Instantly share code, notes, and snippets.

@andir

andir/default.nix Secret

Created May 28, 2020 00:42
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save andir/74869877b30e26c3d851ae27a4d8d2bf to your computer and use it in GitHub Desktop.
Save andir/74869877b30e26c3d851ae27a4d8d2bf to your computer and use it in GitHub Desktop.
Download ZIP
Raw
default.nix
{ pkgs, lib, config, ... }:
with lib;
let
cfg = config.router;
addressesOptions = types.submodule {
options = {
address = mkOption {
type = types.str;
};
prefixLength = mkOption {
type = types.int;
};
};
};
downstreamInterfaceOptions = types.submodule {
options = {
interface = mkOption {
type = types.str;
};
subnetId = mkOption {
# type = types.nullOr types.int;
default = null;
};
v6Addresses = mkOption {
type = types.listOf addressesOptions;
};
v4Addresses = mkOption {
type = types.listOf addressesOptions;
};
};
};
in {
options.router = {
enable = mkEnableOption "enable the router module";
upstreamInterfaces = mkOption {
description = "upstream interfaces";
type = types.listOf types.str;
};
downstreamInterfaces = mkOption {
description = "client interfaces";
type = types.listOf downstreamInterfaceOptions;
};
};
config = mkIf cfg.enable {
# ensure we have a few sane network debugging tools available
environment.systemPackages = with pkgs; [ tcpdump dnstracer ];
programs.mtr.enable = true;
# Can not use this as it pulls in a bunch of garbage rules
networking.useNetworkd = false;
networking.dhcpcd.enable = mkForce false;
boot.kernel.sysctl = {
"net.ipv4.conf.all.forwarding" = 1;
"net.ipv6.conf.all.forwarding" = 1;
};
systemd.services."systemd-networkd".environment.SYSTEMD_LOG_LEVEL = "debug";
systemd.network = let
mkUpstreamIfConfig = name: nameValuePair "00-${name}" {
enable = true;
networkConfig = {
Description = "Upstream network config for ${name}";
IPv6AcceptRA = true;
#IPMasquerade = true; # FIXME: hows does that work on the inside?
#IPForward = "yes";
DHCP = "yes";
#IPv6PrefixDelegation = "dhcpv6";
};
linkConfig = {
RequiredForOnline = "routable";
};
matchConfig = {
Name = name;
};
dhcpV4Config = {
UseDNS = false;
UseRoutes = true;
};
dhcpV6Config = {
PrefixDelegationHint= "::/1";
};
ipv6PrefixDelegationConfig = {
Managed = true;
OtherInformation = true;
};
};
mkClientIfConfig = conf: let v = nameValuePair "00-${conf.interface}" {
enable = true;
matchConfig = {
Name = conf.interface;
};
addresses = (map (addr:
{ addressConfig.Address = "${addr.address}/${toString addr.prefixLength}"; }
) (conf.v4Addresses ++ conf.v6Addresses))
++ [ { addressConfig.Address = "::/64"; } ];
networkConfig = {
DHCPServer = true;
IPv6PrefixDelegation = "dhcpv6";
};
dhcpServerConfig = {
PoolOffset = 10;
EmitDNS = true;
EmitNTP = true;
EmitRouter = true;
EmitTimezone = true;
};
ipv6PrefixDelegationConfig = {
RouterLifetimeSec = 300;
EmitDNS = true;
};
extraConfig = if (conf.subnetId != null) then ''
[Network]
IPv6PDSubnetId=${toString conf.subnetId}
'' else "";
ipv6Prefixes = [
{
ipv6PrefixConfig = {
AddressAutoconfiguration = true;
PreferredLifetimeSec = 1800;
ValidLifetimeSec = 1800;
};
}
];
}; in builtins.trace "${builtins.toJSON v.value}" v;
upstreamConfig = builtins.listToAttrs (map mkUpstreamIfConfig cfg.upstreamInterfaces);
downstreamConfig = builtins.listToAttrs (map mkClientIfConfig cfg.downstreamInterfaces);
in {
enable = true;
networks = mkMerge [
upstreamConfig
downstreamConfig
{
"99-main" = {
networkConfig = {
IPv6AcceptRA = lib.mkForce false;
DHCP = lib.mkForce "no";
};
linkConfig = {
Unmanaged = lib.mkForce "yes";
};
};
}
];
};
};
}
Raw
test.nix
let
makeTest = import (<nixpkgs> + "/nixos/tests/make-test-python.nix");
numVlans = 100;
in makeTest ({pkgs, lib, ...}: let
getIPv4 = nodes: node: iface:
(builtins.head nodes.${node}.config.networking.interfaces.${iface}.ipv4.addresses).address;
getIPv6 = nodes: node: iface:
(builtins.head nodes.${node}.config.networking.interfaces.${iface}.ipv6.addresses).address;
in {
nodes ={
dns = { nodes, ...}: {
virtualisation.vlans = [ 4 ];
networking.dhcpcd.enable = false;
networking.interfaces.eth0.ipv4.addresses = lib.mkForce [];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{ address = "100.65.0.2"; prefixLength = 24; }
];
networking.interfaces.eth1.ipv6.addresses = lib.mkForce [
{ address = "2001:DB9::2"; prefixLength = 126; }
];
networking.interfaces.eth1.ipv4.routes = [
{ address = "100.64.100.0"; prefixLength = 24; via = "100.65.0.1"; }
];
networking.interfaces.eth1.ipv6.routes = [
{ address = "2001:DB8::"; prefixLength = 32; via = "2001:DB9::1"; }
];
};
# The upstream router providing DHPCv4, DHCPv6 and radvd
# It also serves as upstream router
isp = {nodes, pkgs, ...}:{
virtualisation.vlans = [ 1 4 ];
networking.dhcpcd.enable = false;
networking.interfaces.eth0.ipv4.addresses = lib.mkForce [];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{ address = "100.64.100.1"; prefixLength = 24; }
];
networking.interfaces.eth1.ipv6.addresses = lib.mkForce [
{ address = "2001:DB8::"; prefixLength = 64; }
];
networking.interfaces.eth2.ipv4.addresses = lib.mkForce [
{ address = "100.65.0.1"; prefixLength = 24; }
];
networking.interfaces.eth2.ipv6.addresses = lib.mkForce [
{ address = "2001:DB9::1"; prefixLength = 126; }
];
networking.firewall.enable = false;
boot.kernel.sysctl."net.ipv4.conf.all.forwarding" = 1;
boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = 1;
environment.systemPackages = [ pkgs.tcpdump ];
# don't do this in production, this shouldn't just be executable by anyone
security.wrappers.add-dhcpd-lease = {
source = pkgs.writeScript "add-dhcpd-lease" ''
#!${pkgs.stdenv.shell}
exec ${pkgs.iproute}/bin/ip -6 route replace "$1" via "$2"
'';
capabilities = "cap_net_admin+ep";
};
services = {
dhcpd4 = {
enable = true;
interfaces = [ "eth1" ];
extraConfig = ''
option subnet-mask 255.255.255.255;
option routers ${getIPv4 nodes "isp" "eth1"};
option domain-name-servers ${getIPv4 nodes "dns" "eth1"};
option domain-name "customers.acme.org";
subnet 100.64.100.0 netmask 255.255.255.0 {
range 100.64.100.10 100.64.100.100;
}
'';
};
dhcpd6 = {
enable = true;
interfaces = [ "eth1" ];
extraConfig = ''
#default-lease-time 1800;
#max-lease-time 1800;
#min-lease-time 1800
#subnet6 2001:DB8::/2 {
# range6 2001:DB8:0000:0000:ffff:: 2001:DB8:0000:0000:ffff::ffff;
# #prefix6 2001:DB8:f000:: 2001:DB8:ffff:: /48;
# prefix6 0:: 1000:: /1;
#}
subnet6 ::/1 {
range6 ffff:: ffff:ffff:0000:0000:ffff::ffff;
#prefix6 2001:DB8:f000:: 2001:DB8:ffff:: /48;
prefix6 0:: 1000:: /1;
}
on commit {
set IP = pick-first-value(binary-to-ascii(16, 16, ":", substring(option dhcp6.ia-na, 16, 16)), "n/a");
set Prefix = pick-first-value(binary-to-ascii(16, 16, ":", suffix(option dhcp6.ia-pd, 16)), "n/a");
set PrefixLength = pick-first-value(binary-to-ascii(10, 8, ":", substring(suffix(option dhcp6.ia-pd, 17), 0, 1)), "n/a");
log(concat(IP, " ", Prefix, " ", PrefixLength));
execute("/run/wrappers/bin/add-dhcpd-lease", concat(Prefix,"/",PrefixLength), IP);
}
'';
};
radvd = {
enable = true;
config = ''
interface eth1 {
AdvSendAdvert on;
AdvManagedFlag on;
AdvOtherConfigFlag on;
prefix ::/64 {
AdvOnLink on;
AdvAutonomous on;
};
};
'';
};
};
};
router = { pkgs, lib, ... }: {
virtualisation.vlans = [ 1 2 3 7 10 ];
imports = [ ./default.nix ];
networking.interfaces.eth0.ipv4.addresses = lib.mkForce [];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [];
networking.interfaces.eth2.ipv4.addresses = lib.mkForce [];
networking.interfaces.eth3.ipv4.addresses = lib.mkForce [];
networking.interfaces.eth4.ipv4.addresses = lib.mkForce [];
networking.firewall.enable = false;
environment.systemPackages = [ pkgs.iptables ];
systemd.package = pkgs.systemd.overrideAttrs ({ patches ? [], postPatch ? "", ... }: {
patches = patches ++ [ ./systemd-ipv6-pd-prefix.patch ./safe_atoux64.patch ];
postPatch = postPatch + ''
sed -e 's/128\*1024\*1024/1024*1024*1024/g' -i src/network/networkd-manager.c
grep RCVBUF_SIZE src/network/networkd-manager.c
'';
});
systemd.network.netdevs = lib.listToAttrs (lib.genList (n: lib.nameValuePair "10-vlan${toString (10 +n)}" {
netdevConfig = {
Name = "vlan${toString (10 +n)}";
Kind = "vlan";
};
vlanConfig = {
Id = n;
};
}) numVlans);
systemd.network.networks."00-vlans" = {
matchConfig.Name = "eth5";
vlan = lib.genList (n: "vlan${toString (n+10)}") numVlans;
};
router = {
enable = true;
upstreamInterfaces = [ "eth1" ];
downstreamInterfaces = [
{
interface = "eth2";
# subnetId = "0x12ff";
v4Addresses = [ { address = "192.168.42.1"; prefixLength = 24; } ];
v6Addresses = [ { address = "fd42::1"; prefixLength = 64; } ];
}
{
interface = "eth4";
subnetId = "8000000000000000";
v4Addresses = [ { address = "192.168.43.1"; prefixLength = 24; } ];
v6Addresses = [ { address = "fd43::1"; prefixLength = 64; } ];
}
] ++ (lib.genList (n: {
interface = "vlan${toString (10 + n)}";
subnetId = "0x${toString n}";
v4Addresses = [ ];
v6Addresses = [ ];
}) numVlans);
};
};
client = {pkgs, ...}: {
virtualisation.vlans = [ 2 ];
networking.interfaces.eth0.ipv4.addresses = lib.mkForce [];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [];
networking.interfaces.eth0.ipv6.addresses = lib.mkForce [];
networking.interfaces.eth1.ipv6.addresses = lib.mkForce [];
networking.interfaces.eth0.useDHCP = false;
boot.kernel.sysctl."net.ipv6.conf.eth0.accept_ra" = 0;
programs.mtr.enable = true;
# networking.useDHCP = true;
# networking.dhcpcd = {
# enable = true;
# denyInterfaces = [ "eth0" ];
# allowInterfaces = [ "eth1" ];
# };
};
client2 = {pkgs, ...}: {
virtualisation.vlans = [ 7 ];
networking.interfaces.eth0.ipv4.addresses = lib.mkForce [];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [];
networking.interfaces.eth0.ipv6.addresses = lib.mkForce [];
networking.interfaces.eth1.ipv6.addresses = lib.mkForce [];
networking.interfaces.eth0.useDHCP = false;
boot.kernel.sysctl."net.ipv6.conf.eth0.accept_ra" = 0;
programs.mtr.enable = true;
# networking.useDHCP = true;
# networking.dhcpcd = {
# enable = true;
# denyInterfaces = [ "eth0" ];
# allowInterfaces = [ "eth1" ];
# };
};
};
testScript = { nodes }: ''
router.start()
isp.start()
isp.wait_for_unit("multi-user.target")
router.wait_for_unit("multi-user.target")
isp.execute("sleep 90")
print(router.succeed("networkctl status"))
print(router.succeed("ip -6 r"))
# import json
# def assert_forwarding(machine, iface):
# # I FUCKING HATE BLACK... This code is less readable after the forced formatting... 🤬
# forwarding = machine.succeed(
# f"cat /proc/sys/net/ipv4/conf/{iface}/forwarding"
# ).strip()
# assert forwarding == "1"
# forwarding = machine.succeed(
# f"cat /proc/sys/net/ipv6/conf/{iface}/forwarding"
# ).strip()
# assert forwarding == "1"
# isp.start()
# dns.start()
# isp.wait_for_unit("multi-user.target")
# dns.wait_for_unit("multi-user.target")
# isp.succeed("sleep 15")
## subtest "The upstream router and DNS server should be able to reach each other", sub {
# isp.succeed("ping -c1 ${getIPv4 nodes "dns" "eth1" }")
# dns.succeed("ping -c1 ${getIPv4 nodes "isp" "eth2" }")
# isp.log(isp.succeed("ping -c1 2001:DB9::2 2>&1"))
## }
## subtest "The upstream router should be good and well. No failed DHCPd", sub {
# isp.wait_for_unit("dhcpd4")
# isp.wait_for_unit("dhcpd6")
# isp.wait_for_unit("radvd")
# assert_forwarding(isp, "eth1")
# assert_forwarding(isp, "eth2")
## }
# router.start()
# router.wait_for_unit("multi-user.target")
## FIXME: we sleep since networkd doesn't consider IPv6 required for the network-online.target…
# router.succeed("sleep 15")
## subtest "The router should receive v4 leases from upstream", sub {
# router.wait_for_unit("network-online.target")
# router.succeed("sleep 2")
# router.succeed("ip a show dev eth1")
# router_ip_address_config = json.loads(router.succeed("ip --json a show dev eth1 2>&1"))
# router.log(str(router_ip_address_config))
# router.log(router.succeed("networkctl status -a 2>&1"))
# router.log(router.succeed("ip -4 r 2>&1"))
# router.log(router.succeed("ip -6 r 2>&1"))
# router.log("trying to ping isp")
# router.succeed("ping -c1 ${getIPv4 nodes "isp" "eth2" } 2>&1")
# router.log("trying to ping dns")
# router.succeed("ping -c1 ${getIPv4 nodes "dns" "eth1" } 2>&1")
# router.log(router.succeed("ss -lpn | grep systemd-network 3>&1"))
# assert_forwarding(router, "eth1")
# assert_forwarding(router, "eth2")
## enable NAT for our client interface, networkd is kinda stupid in this
## regard as it only supported source based NATs
# router.succeed("iptables -t nat -A POSTROUTING -o eth1 -j MASQUERADE")
# client.start()
# client.wait_for_unit("network-online.target")
## FIXME: we sleep since networkd doesn't consider IPv6 required for the network-online.target…
# router.succeed("sleep 15")
# client.log(client.succeed("ip a 2>&1"))
# client.log(client.succeed("ip -4 r 2>&1"))
# client.log(client.succeed("ip -6 r 2>&1"))
# client.log(client.succeed("ping -c1 ${getIPv4 nodes "isp" "eth2" } 2>&1"))
# client.log(client.succeed("ping -c1 ${getIPv6 nodes "isp" "eth2" } 2>&1"))
# client.log(client.succeed("ping -c1 ${getIPv4 nodes "dns" "eth1" } 2>&1"))
# client.log(client.succeed("ping -c1 ${getIPv6 nodes "dns" "eth1" } 2>&1"))
## $client->log($client->succeed("mtr -6 --report 2001:DB8:: 2>&1"))
# client.log(client.succeed("ping -c1 2001:DB9::2 2>&1"))
# client.log(client.succeed("mtr -6 --report 2001:DB9::2 2>&1"))
'';
})
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment