Set WAN DHCP client Set LAN static IP and subnet
cat /etc/network/interfaces
source /etc/network/interfaces.d/*
# The loopback network interface
auto lo
iface lo inet loopback
# DHCP config for ens4 (WAN)
auto ens4
iface ens4 inet dhcp
# Static config for ens5 (LAN)
auto ens5
iface ens5 inet static
address 10.40.0.1
netmask 255.255.254.0
# gateway 192.168.1.1
# dns-nameservers 1.1.1.1
ip link set ens4 up
ip link set ens5 up
apt update && apt install nftables -y
sudo systemctl enable nftables
sudo systemctl restart networking
And add to /etc/sysctl.conf
echo 1 > /proc/sys/net/ipv4/ip_forward
#!/usr/sbin/nft -f
flush ruleset
define DEV_PRIVATE = ens5
define DEV_WORLD = ens4
define NET_PRIVATE = 10.40.0.0/23
table ip global {
chain inbound_world {
# accepting ping (icmp-echo-request) for diagnostic purposes.
# However, it also lets probes discover this host is alive.
# This sample accepts them within a certain rate limit:
#
# icmp type echo-request limit rate 5/second accept
# allow SSH connections from some well-known internet host
# ip saddr 81.209.165.42 tcp dport ssh accept
}
chain inbound_private {
# accepting ping (icmp-echo-request) for diagnostic purposes.
icmp type echo-request limit rate 5/second accept
# allow DHCP, DNS and SSH from the private network
ip protocol . th dport vmap { tcp . 22 : accept, udp . 53 : accept, tcp . 53 : accept, udp . 67 : accept}
}
chain inbound {
type filter hook input priority 0; policy drop;
# Allow traffic from established and related packets, drop invalid
ct state vmap { established : accept, related : accept, invalid : drop }
# allow loopback traffic, anything else jump to chain for further evaluation
iifname vmap { lo : accept, $DEV_WORLD : jump inbound_world, $DEV_PRIVATE : jump inbound_private }
# the rest is dropped by the above policy
}
chain forward {
type filter hook forward priority 0; policy drop;
# Allow traffic from established and related packets, drop invalid
ct state vmap { established : accept, related : accept, invalid : drop }
# connections from the internal net to the internet or to other
# internal nets are allowed
iifname $DEV_PRIVATE accept
# the rest is dropped by the above policy
}
chain postrouting {
type nat hook postrouting priority 100; policy accept;
# masquerade private IP addresses
ip saddr $NET_PRIVATE oifname $DEV_WORLD masquerade
}
}
apt install kea-dhcp4-server -y
mv kea-dhcp4.conf kea-dhcp4.sample.conf && vim kea-dhcp4.conf
# /etc/kea/kea-dhcp4.conf
{
# DHCPv4 configuration starts on the next line
"Dhcp4": {
# Set up global values that apply to leases
"valid-lifetime": 1296000,
"renew-timer": 864000,
"rebind-timer": 864000,
# Set up reclaimation for expired leases and leases affinity
"expired-leases-processing": {
"reclaim-timer-wait-time": 10,
"flush-reclaimed-timer-wait-time": 25,
"hold-reclaimed-time": 3600,
"max-reclaim-leases": 100,
"max-reclaim-time": 250,
"unwarned-reclaim-cycles": 5
},
# Set up the interfaces to be used by the server.
"interfaces-config": {
"interfaces": [ "enp5s0" ]
},
# Specify the type of lease database
"lease-database": {
"type": "memfile",
"persist": true,
"name": "/home/_kea/dhcp4.leases",
"lfc-interval": 3600
},
# Specify the subnets from which we will be leasing addresses.
"subnet4": [
{
"subnet": "10.40.0.0/23",
"pools": [
{
"pool": "10.40.0.100 - 10.40.1.250"
}
],
"option-data": [
{
# For each IPv4 subnet you most likely need to specify
# at least one router.
"name": "routers",
"data": "10.40.0.1"
},
{
# For each IPv4 subnet you most likely need to specify
# at least one router.
"name": "domain-name",
"data": "home.arpa"
},
{
# For each IPv4 subnet you most likely need to specify
# at least one router.
"name": "domain-name-servers",
"data": "1.1.1.1, 8.8.8.8"
}
],
"reservations": [
{
# Use this section to specify the target board's fixed
# IP address
# Substitute the target board MAC address
"client-id": "00:1b:41:0a:00:3d",
# Set the static IP address for the target board
"ip-address": "10.40.0.5",
"hostname": "david-nas"
}
]
}
]
# DHCPv4 configuration ends with the next line
}
}
ToDo: Figure out apparmor issues. Had to disable to let dhcpd run. It didn't like kea reading the dhcp leases cache file for whatever reason. Evidently it's a known issue from Google searches.
apt install apparmor-utils
aa-disable /etc/apparmor.d/usr.sbin.kea-dhcp4
systemctl start kea-dhcp4-server
Sadly, this approach wasn't able to deliver the performance I was after. Using 2.5gbe interfaces, I was hoping for line-rate NAT with a single stream. Here's what I ended up with. Scenario: downloading a file from the internet. Actual setup" client on LAN interface of debian router. WAN of the debian router is plugged into my network as just another client. iperf server host is a truenas scale instance with 10gbe (capable of 10gbps single stream).
admin@david-nas[~]$ iperf3 -c truenas-scale.seas.house -R
Connecting to host truenas-scale.seas.house, port 5201
Reverse mode, remote host truenas-scale.seas.house is sending
[ 5] local 10.40.0.100 port 49272 connected to 10.23.4.21 port 5201
[ ID] Interval Transfer Bitrate
[ 5] 0.00-1.00 sec 175 MBytes 1.46 Gbits/sec
[ 5] 1.00-2.00 sec 140 MBytes 1.18 Gbits/sec
[ 5] 2.00-3.00 sec 142 MBytes 1.19 Gbits/sec
[ 5] 3.00-4.00 sec 140 MBytes 1.17 Gbits/sec
[ 5] 4.00-5.00 sec 146 MBytes 1.22 Gbits/sec
[ 5] 5.00-6.00 sec 139 MBytes 1.16 Gbits/sec
[ 5] 6.00-7.00 sec 157 MBytes 1.31 Gbits/sec
[ 5] 7.00-8.00 sec 140 MBytes 1.17 Gbits/sec
[ 5] 8.00-9.00 sec 155 MBytes 1.30 Gbits/sec
[ 5] 9.00-10.00 sec 140 MBytes 1.18 Gbits/sec
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval Transfer Bitrate Retr
[ 5] 0.00-10.04 sec 1.44 GBytes 1.23 Gbits/sec 10437 sender
[ 5] 0.00-10.00 sec 1.44 GBytes 1.24 Gbits/sec receiver
Bumping up to 6 streams nets us 1.83 Gbits/sec. Still not quite the 2.5gbe we are after.
iperf3 -c truenas-scale.seas.house -R -P 6
[ ID] Interval Transfer Bitrate Retr
[ 5] 0.00-10.04 sec 416 MBytes 347 Mbits/sec 4579 sender
[ 5] 0.00-10.00 sec 414 MBytes 348 Mbits/sec receiver
[ 7] 0.00-10.04 sec 433 MBytes 362 Mbits/sec 4446 sender
[ 7] 0.00-10.00 sec 432 MBytes 363 Mbits/sec receiver
[ 9] 0.00-10.04 sec 316 MBytes 264 Mbits/sec 3607 sender
[ 9] 0.00-10.00 sec 314 MBytes 264 Mbits/sec receiver
[ 11] 0.00-10.04 sec 354 MBytes 296 Mbits/sec 4249 sender
[ 11] 0.00-10.00 sec 353 MBytes 296 Mbits/sec receiver
[ 13] 0.00-10.04 sec 319 MBytes 267 Mbits/sec 3964 sender
[ 13] 0.00-10.00 sec 318 MBytes 267 Mbits/sec receiver
[ 15] 0.00-10.04 sec 349 MBytes 291 Mbits/sec 4450 sender
[ 15] 0.00-10.00 sec 348 MBytes 292 Mbits/sec receiver
[SUM] 0.00-10.04 sec 2.13 GBytes 1.83 Gbits/sec 25295 sender
[SUM] 0.00-10.00 sec 2.13 GBytes 1.83 Gbits/sec receiver
With 12 streams we're able to finally hit a little over 2 gig.
iperf3 -c truenas-scale.seas.house -R -P 12
[SUM] 0.00-10.03 sec 2.44 GBytes 2.09 Gbits/sec 36266 sender
[SUM] 0.00-10.00 sec 2.43 GBytes 2.09 Gbits/sec receiver
This was the most performance I could get out of this setup for NAT resembling the real-world usecase of a soho router.
On the other hand, other scenarios are able to achieve near line-rate 2.5gbe:
Regular old upload to the server hits 2.26 - nearly the real-world maximum we can expect after factoring in overhead.
admin@david-nas[~]$ iperf3 -c truenas-scale.seas.house
Connecting to host truenas-scale.seas.house, port 5201
[ 5] local 10.40.0.100 port 48534 connected to 10.23.4.21 port 5201
[ ID] Interval Transfer Bitrate Retr Cwnd
[ 5] 0.00-1.00 sec 272 MBytes 2.28 Gbits/sec 1 626 KBytes
[ 5] 1.00-2.00 sec 271 MBytes 2.27 Gbits/sec 0 638 KBytes
[ 5] 2.00-3.00 sec 270 MBytes 2.27 Gbits/sec 0 649 KBytes
[ 5] 3.00-4.00 sec 270 MBytes 2.26 Gbits/sec 1 632 KBytes
[ 5] 4.00-5.00 sec 270 MBytes 2.26 Gbits/sec 2 556 KBytes
[ 5] 5.00-6.00 sec 270 MBytes 2.26 Gbits/sec 2 624 KBytes
[ 5] 6.00-7.00 sec 270 MBytes 2.26 Gbits/sec 0 643 KBytes
[ 5] 7.00-8.00 sec 271 MBytes 2.28 Gbits/sec 0 649 KBytes
[ 5] 8.00-9.00 sec 268 MBytes 2.24 Gbits/sec 1 619 KBytes
[ 5] 9.00-10.00 sec 271 MBytes 2.27 Gbits/sec 0 660 KBytes
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval Transfer Bitrate Retr
[ 5] 0.00-10.00 sec 2.64 GBytes 2.27 Gbits/sec 7 sender
[ 5] 0.00-10.04 sec 2.64 GBytes 2.26 Gbits/sec receiver
Transfers with the router itself are also line-rate capable. So if you wanted to run a single box with a bunch of homelab stuff, plex, hw transcoding, containers, storage, even a full debian desktop with gpu accelerated rendering over remote desktop... It could also be your router. If your internet is <1gbps you'd never notice the nat bottleneck.
admin@david-nas[~]$ iperf3 -c 10.40.0.1
Connecting to host 10.40.0.1, port 5201
[ 5] local 10.40.0.100 port 53792 connected to 10.40.0.1 port 5201
[ ID] Interval Transfer Bitrate Retr Cwnd
[ 5] 0.00-1.00 sec 273 MBytes 2.29 Gbits/sec 0 584 KBytes
[ 5] 1.00-2.00 sec 271 MBytes 2.27 Gbits/sec 0 649 KBytes
[ 5] 2.00-3.00 sec 271 MBytes 2.28 Gbits/sec 0 693 KBytes
[ 5] 3.00-4.00 sec 270 MBytes 2.26 Gbits/sec 1 580 KBytes
[ 5] 4.00-5.00 sec 271 MBytes 2.28 Gbits/sec 0 588 KBytes
[ 5] 5.00-6.00 sec 271 MBytes 2.27 Gbits/sec 0 645 KBytes
[ 5] 6.00-7.00 sec 270 MBytes 2.27 Gbits/sec 0 645 KBytes
[ 5] 7.00-8.00 sec 271 MBytes 2.28 Gbits/sec 0 645 KBytes
[ 5] 8.00-9.00 sec 270 MBytes 2.26 Gbits/sec 0 645 KBytes
[ 5] 9.00-10.00 sec 271 MBytes 2.28 Gbits/sec 0 645 KBytes
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval Transfer Bitrate Retr
[ 5] 0.00-10.00 sec 2.65 GBytes 2.27 Gbits/sec 1 sender
[ 5] 0.00-10.00 sec 2.64 GBytes 2.27 Gbits/sec receiver
As far as system load on the router, it never gets high: load average: 0.08, 0.07, 0.03 is what I observed while running any of these tests. There's no load whatsoever. But there are other clues:
When not bottlenecked we get:
admin@david-nas[~]$ iperf3 -c truenas-scale.seas.house -t 0
Router: %Cpu(s): 0.0 us, 0.0 sy, 0.0 ni, 93.6 id, 0.0 wa, 0.0 hi, 6.4 si, 0.0 st
When using many streams to get around the download-direction bottleneck:
admin@david-nas[~]$ iperf3 -c truenas-scale.seas.house -t 0 -R -P 8
Router: %Cpu(s): 0.1 us, 0.1 sy, 0.0 ni, 96.4 id, 0.0 wa, 0.0 hi, 3.3 si, 0.0 st
When bottlenecked:
iperf3 -c truenas-scale.seas.house -t 0 -R
Router: %Cpu(s): 0.0 us, 0.1 sy, 0.0 ni, 97.5 id, 0.0 wa, 0.0 hi, 2.4 si, 0.0 st
There's more to this that I don't think I'll ever fully understand.
At this point I've seen what I need to see. This started out as an effort to recycle my inlaws' old Dell into something useful. I really wanted to see how nftables can perform. But it's a lot to set up and manage, when I really just need something simple that won't bottleneck a "multi gig" WAN connection. I think I'll just install pfsense and call it a day.
Here's some info about the hardware I tried this on: Dell XPS desktop from 2011ish.
inxi -Fxz
System:
Kernel: 6.1.0-9-amd64 arch: x86_64 bits: 64 compiler: gcc v: 12.2.0 Console: pty pts/2
Distro: Debian GNU/Linux 12 (bookworm)
Machine:
Type: Desktop System: Dell product: XPS 8500 v: N/A serial: <filter>
Mobo: Dell model: 0NW73C v: A00 serial: <filter> UEFI: Dell v: A09 date: 09/05/2012
CPU:
Info: quad core model: Intel Core i7-3770 bits: 64 type: MT MCP arch: Ivy Bridge rev: 9 cache:
L1: 256 KiB L2: 1024 KiB L3: 8 MiB
Speed (MHz): avg: 1775 high: 3000 min/max: 1600/3900 cores: 1: 1600 2: 1600 3: 3000 4: 1600
5: 1600 6: 1600 7: 1600 8: 1600 bogomips: 54271
Flags: avx ht lm nx pae sse sse2 sse3 sse4_1 sse4_2 ssse3 vmx
Graphics:
Device-1: AMD Pitcairn XT [Radeon HD 7870 GHz Edition] driver: radeon v: kernel arch: GCN-1
bus-ID: 01:00.0 temp: 32.0 C
Display: x11 server: X.org v: 1.21.1.7 with: Xwayland v: 22.1.9 driver: X: loaded: radeon
unloaded: fbdev,modesetting,vesa dri: radeonsi gpu: radeon tty: 232x96
API: OpenGL Message: GL data unavailable in console for root.
Audio:
Device-1: AMD Oland/Hainan/Cape Verde/Pitcairn HDMI Audio [Radeon HD 7000 Series]
driver: snd_hda_intel v: kernel bus-ID: 01:00.1
API: ALSA v: k6.1.0-9-amd64 status: kernel-api
Server-1: PipeWire v: 0.3.65 status: n/a (root, process)
Server-2: PulseAudio v: 16.1 status: off (using pipewire-pulse)
Network:
Device-1: Qualcomm Atheros AR9485 Wireless Network Adapter vendor: Dell driver: ath9k v: kernel
bus-ID: 02:00.0
IF: wlp2s0 state: down mac: <filter>
Device-2: Intel Ethernet I225-V driver: igc v: kernel port: N/A bus-ID: 03:00.0
IF: enp3s0 state: up speed: 2500 Mbps duplex: full mac: <filter>
Device-3: Realtek RTL8111/8168/8411 PCI Express Gigabit Ethernet vendor: Dell driver: r8169
v: kernel port: d000 bus-ID: 04:00.0
IF: enp4s0 state: up speed: 1000 Mbps duplex: full mac: <filter>
Device-4: Intel Ethernet I225-V driver: igc v: kernel port: N/A bus-ID: 05:00.0
IF: enp5s0 state: up speed: 2500 Mbps duplex: full mac: <filter>
Bluetooth:
Device-1: Qualcomm Atheros Bluetooth USB Host Controller type: USB driver: btusb v: 0.8
bus-ID: 1-1.5:3
Report: hciconfig ID: hci0 rfk-id: 0 state: up address: <filter> bt-v: 2.1 lmp-v: 4.0
Drives:
Local Storage: total: 29.82 GiB used: 10.73 GiB (36.0%)
ID-1: /dev/sda vendor: Samsung model: SSD PM830 mSATA 32GB size: 29.82 GiB
Partition:
ID-1: / size: 28.89 GiB used: 10.72 GiB (37.1%) fs: ext4 dev: /dev/sda2
ID-2: /boot/efi size: 299.4 MiB used: 5.8 MiB (1.9%) fs: vfat dev: /dev/sda1
Swap:
Alert: No swap data was found.
Sensors:
System Temperatures: cpu: 29.0 C mobo: N/A gpu: radeon temp: 32.0 C
Fan Speeds (RPM): cpu: 881 mobo: 909
Info:
Processes: 188 Uptime: 1h 50m Memory: 23.43 GiB used: 833.2 MiB (3.5%) Init: systemd
target: graphical (5) Compilers: gcc: 12.2.0 Packages: 2509 Shell: Bash v: 5.2.15 inxi: 3.3.26
root@old-dell-white:~# lscpu
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Address sizes: 36 bits physical, 48 bits virtual
Byte Order: Little Endian
CPU(s): 8
On-line CPU(s) list: 0-7
Vendor ID: GenuineIntel
BIOS Vendor ID: Intel
Model name: Intel(R) Core(TM) i7-3770 CPU @ 3.40GHz
BIOS Model name: Intel(R) Core(TM) i7-3770 CPU @ 3.40GHz Fill By OEM CPU @ 3.4GHz
BIOS CPU family: 6
CPU family: 6
Model: 58
Thread(s) per core: 2
Core(s) per socket: 4
Socket(s): 1
Stepping: 9
CPU(s) scaling MHz: 43%
CPU max MHz: 3900.0000
CPU min MHz: 1600.0000
BogoMIPS: 6783.97
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop _tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm cpuid_fault epb pti ssbd i
brs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase smep erms xsaveopt dtherm ida arat pln pts md_clear flush_l1d
Virtualization features:
Virtualization: VT-x
Caches (sum of all):
L1d: 128 KiB (4 instances)
L1i: 128 KiB (4 instances)
L2: 1 MiB (4 instances)
L3: 8 MiB (1 instance)
NUMA:
NUMA node(s): 1
NUMA node0 CPU(s): 0-7
Vulnerabilities:
Itlb multihit: KVM: Mitigation: VMX disabled
L1tf: Mitigation; PTE Inversion; VMX conditional cache flushes, SMT vulnerable
Mds: Mitigation; Clear CPU buffers; SMT vulnerable
Meltdown: Mitigation; PTI
Mmio stale data: Unknown: No mitigations
Retbleed: Not affected
Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl
Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization
Spectre v2: Mitigation; Retpolines, IBPB conditional, IBRS_FW, STIBP conditional, RSB filling, PBRSB-eIBRS Not affected
Srbds: Vulnerable: No microcode
Tsx async abort: Not affected
root@old-dell-white:~# dmidecode -t memory
# dmidecode 3.4
Getting SMBIOS data from sysfs.
SMBIOS 2.7 present.
Handle 0x002A, DMI type 16, 23 bytes
Physical Memory Array
Location: System Board Or Motherboard
Use: System Memory
Error Correction Type: None
Maximum Capacity: 32 GB
Error Information Handle: Not Provided
Number Of Devices: 4
Handle 0x002C, DMI type 17, 34 bytes
Memory Device
Array Handle: 0x002A
Error Information Handle: Not Provided
Total Width: 64 bits
Data Width: 64 bits
Size: 8 GB
Form Factor: DIMM
Set: None
Locator: DIMM3
Bank Locator: BANK 0
Type: DDR3
Type Detail: Synchronous
Speed: 1600 MT/s
Manufacturer: Micron
Serial Number: DE9A77BD
Asset Tag: 9876543210
Part Number: 16JTF1G64AZ-1G6D1
Rank: 2
Configured Memory Speed: 1600 MT/s
Handle 0x002E, DMI type 17, 34 bytes
Memory Device
Array Handle: 0x002A
Error Information Handle: Not Provided
Total Width: 64 bits
Data Width: 64 bits
Size: 8 GB
Form Factor: DIMM
Set: None
Locator: DIMM1
Bank Locator: BANK 1
Type: DDR3
Type Detail: Synchronous
Speed: 1600 MT/s
Manufacturer: Micron
Serial Number: DE9A77B0
Asset Tag: 9876543210
Part Number: 16JTF1G64AZ-1G6D1
Rank: 2
Configured Memory Speed: 1600 MT/s
Handle 0x0030, DMI type 17, 34 bytes
Memory Device
Array Handle: 0x002A
Error Information Handle: Not Provided
Total Width: Unknown
Data Width: Unknown
Size: No Module Installed
Form Factor: DIMM
Set: None
Locator:
Bank Locator:
Type: Unknown
Type Detail: None
Handle 0x0031, DMI type 17, 34 bytes
Memory Device
Array Handle: 0x002A
Error Information Handle: Not Provided
Total Width: 64 bits
Data Width: 64 bits
Size: 8 GB
Form Factor: DIMM
Set: None
Locator: DIMM2
Bank Locator: BANK 3
Type: DDR3
Type Detail: Synchronous
Speed: 1600 MT/s
Manufacturer: Micron
Serial Number: DE9A77B1
Asset Tag: 9876543210
Part Number: 16JTF1G64AZ-1G6D1
Rank: 2
Configured Memory Speed: 1600 MT/s
Well, looks like Pfsense can't hit line speed either on this hardware. Single stream performance in a download scenario manages just 1.5gbps. With 6 streams we can max out the 2.5gbe interface, but that's not what I'm after. Worse yet, pfsense lacks compatible fan control for this antique Dell XPS PC, so the fans run full tilt the entire time creating tons of noise and sucking up dust.
The quest for a true 2.5gbps router continues.
You probably want something like it. So you can cache DNS requests, and so LAN hosts get listed in DNS.