gistfile1.txt

### HOST /etc/rc.conf ###

r1% cat /etc/rc.conf
clear_tmp_enable="YES"
sendmail_enable="NONE"
hostname="r1"
local_unbound_enable="YES"
sshd_enable="YES"
powerd_enable="YES"
# Set dumpdev to "AUTO" to enable crash dumps, "NO" to disable
dumpdev="AUTO"
#ntpdate_enable="YES"
#ntpdate_flags="-b -4"
#ntpdate_hosts="ntp.belnet.be"
dhcpd_enable="YES"
dhcpd_ifaces="switch-local"
microcode_update_enable="YES"
defaultrouter="192.168.0.254"

jail_enable="YES"
salt_minion_enable="YES"
ipv6_network_interfaces="none"
ip6addrctl_policy="ipv4_prefer"

ifconfig_igb0="-tso4 -tso6 -lro -vlanhwtso up"
ifconfig_igb1="-tso4 -tso6 -lro -vlanhwtso up"
ifconfig_igb2="-tso4 -tso6 -lro -vlanhwtso up"
cloned_interfaces="lagg0 bridge0 bridge1 bridge2"
ifconfig_lagg0="laggproto lacp laggport igb0 laggport igb1 laggport igb2 up"
ifconfig_bridge0_name="switch-modem"
ifconfig_bridge1_name="switch-local"
ifconfig_bridge2_name="switch-pfsync"
vlans_lagg0="vlan10 vlan11 vlan12"
create_args_vlan10="vlan 10"
create_args_vlan11="vlan 11"
create_args_vlan12="vlan 12"
ifconfig_vlan10="up"
ifconfig_vlan11="up"
ifconfig_vlan12="up"
ifconfig_switch_modem="addm vlan10 up"
ifconfig_switch_local="inet 192.168.0.11/24 addm vlan11 up"
ifconfig_switch_pfsync="addm vlan12 up"
jail_list="fw1 salt1 haproxy1"


### HOST /etc/sysctl.conf ###

r1% cat /etc/sysctl.conf 
#net.inet.carp.preempt=1
kern.ipc.soacceptqueue=4096
net.inet.tcp.sendspace=65536
kern.random.harvest.mask=351
kern.ipc.maxsockbuf=4194304
dev.igb.0.fc=0
dev.igb.1.fc=0
dev.igb.2.fc=0
dev.igb.2.iflib.tx_abdicate=1
dev.igb.1.iflib.tx_abdicate=1
dev.igb.0.iflib.tx_abdicate=1
net.inet.ip.redirect=0
net.inet6.ip6.redirect=0
#net.inet.carp.senderr_demotion_factor=0
net.link.bridge.pfil_onlyip=0
net.link.bridge.pfil_bridge=0
net.link.bridge.pfil_member=0
net.link.bridge.log_stp=1


### HOST /etc/jail.conf ###

r1% cat /etc/jail.conf 
exec.system_user = "root";
exec.jail_user = "root";
exec.prestart += "cp /etc/resolv.conf $path/etc";
exec.prestart += "cp /etc/localtime $path/etc";
exec.start = "/bin/sh /etc/rc";
exec.stop += "/bin/sh /etc/rc.shutdown";
exec.poststop += "rm $path/etc/resolv.conf";
exec.poststop += "rm $path/etc/localtime";
exec.clean;

mount.devfs;
mount.fstab = "/etc/fstab.$name";

path = "/jails/$name";
host.hostname = "$name.lan";


salt1 {
ip4.addr = switch-local|192.168.0.20;
}

haproxy1 {
ip4.addr = switch-local|192.168.0.26;
}

fw1 {
vnet;
vnet.interface = "epair0b";
vnet.interface += "epair1b";
vnet.interface += "epair2b";
devfs_ruleset = "501";
allow.raw_sockets = "1";

exec.prestart += "ifconfig epair0 create up";
exec.prestart += "ifconfig epair1 create up";
exec.prestart += "ifconfig epair2 create up";
exec.prestart += "ifconfig epair0a up descr vnet-${name}-modem";
exec.prestart += "ifconfig epair1a up descr vnet-${name}";
exec.prestart += "ifconfig epair2a up descr vnet-${name}-pfsync";
exec.prestart += "ifconfig switch-modem addm epair0a up";
exec.prestart += "ifconfig switch-local addm epair1a up";
exec.prestart += "ifconfig switch-pfsync addm epair2a up";

#exec.start += "/sbin/ifconfig lo0 127.0.0.1 up";
#exec.start += "/sbin/ifconfig epair0b up";
#exec.start += "/sbin/ifconfig epair1b 192.168.0.252/24 up";
#exec.start += "/sbin/route add default 192.168.0.254";

exec.poststop += "ifconfig epair0b -vnet ${name}";
exec.poststop += "ifconfig epair1b -vnet ${name}";
exec.poststop += "ifconfig epair2b -vnet ${name}";
exec.poststop += "ifconfig switch-modem deletem epair0a";
exec.poststop += "ifconfig switch-local deletem epair1a";
exec.poststop += "ifconfig switch-pfsync deletem epair2a";
exec.poststop += "sleep 2";
exec.poststop += "ifconfig epair0a destroy";
exec.poststop += "ifconfig epair1a destroy";
exec.poststop += "ifconfig epair2a destroy";
}


### JAIL fw1 /etc/rc.conf ###

r1% cat /jails/fw1/etc/rc.conf
rpcbind_enable="NO"
syslogd_flags="-ss"
sendmail_enable="NONE"
sshd_enable="YES"
cron_flags="$cron_flags -J 15"
salt_minion_enable="YES"
ifconfig_epair0b_name="modem"
ifconfig_epair1b_name="local"
ifconfig_epair2b_name="pfsync"
ifconfig_modem="up"
ifconfig_local="inet 192.168.0.252/24 up"
ifconfig_local_alias0="inet vhid 11 advskew 50 pass xxx alias 192.168.0.254/32 up"
ifconfig_pfsync="inet 10.10.10.9/30 up"
pf_enable="YES"
pf_rules="/etc/pf.conf"
pflog_enable="YES"
pfsync_enable="YES"
pfsync_syncdev="pfsync"
pfsync_syncpeer="10.10.10.10"
gateway_enable="YES"
ipv6_network_interfaces="none"
ip6addrctl_policy="ipv4_prefer"
gateway_enable="YES"
mpd_enable="YES"
blacklistd_enable="YES"


### JAIL fw1 /etc/sysctl.conf ###

r1% cat /jails/fw1/etc/sysctl.conf
net.inet.carp.preempt=1
net.inet.tcp.sendspace=65536
net.inet.ip.redirect=0
net.inet6.ip6.redirect=0
net.inet.carp.senderr_demotion_factor=0


### JAIL fw1 /etc/pf.conf.conf ###

root@r1:/home/julien # jexec fw1 cat /etc/pf.conf
lo_if="lo0"
int_if="local"
ext_if="tun0"
localnet=$int_if:network:0
martians = "{ 127.0.0.0/8, 192.168.0.0/16, 172.16.0.0/12, 10.0.0.0/8, 169.254.0.0/16, 192.0.2.0/24, 0.0.0.0/8, 240.0.0.0/4 }"

# Options
set fingerprints "/etc/pf.os"
set block-policy return
set debug urgent
set ruleset-optimization basic
set skip on $lo_if
set loginterface none

scrub in all

nat on $int_if from $localnet to mail.lan port { 143, 993 } -> 192.168.0.254
nat on $int_if from $localnet to haproxy1.lan port { 80, 443 } -> 192.168.0.254
nat on $ext_if from $localnet to any -> ($ext_if)

rdr pass on $ext_if proto tcp from !$localnet to any port { 143, 993 } -> mail.lan
rdr pass on $int_if proto tcp from $localnet to 192.168.0.254 port { 143, 993 }-> mail.lan
rdr pass on $int_if proto tcp from $localnet to 192.168.0.254 port { 80, 443 } -> haproxy1.lan
rdr pass on $int_if proto tcp from $localnet to ($ext_if) port 80 -> haproxy1.lan
rdr pass on $ext_if proto tcp from !$localnet to ($ext_if) port { 80, 443 } -> haproxy1.lan
#rdr pass on $ext_if proto tcp from any to any port 443 -> haproxy1.lan port 443

anchor "blacklistd/*" in on $ext_if

block log all
block drop in quick on $ext_if from $martians to any
block drop out quick on $ext_if from any to $martians

antispoof log quick for $int_if
antispoof log quick for $ext_if

pass in proto icmp
pass quick on pfsync proto pfsync keep state (no-sync)
pass quick on local proto carp keep state (no-sync)

pass in on $int_if from any to any
pass in on $ext_if inet proto tcp from any to any port { https, imaps, ssh }

pass out all


### HOST ifconfig ###

r1% ifconfig
igb0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=e120bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,WOL_MAGIC,VLAN_HWFILTER,RXCSUM_IPV6,TXCSUM_IPV6>
        ether 00:0d:b9:4b:10:c0
        media: Ethernet autoselect (1000baseT <full-duplex>)
        status: active
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
igb1: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=e120bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,WOL_MAGIC,VLAN_HWFILTER,RXCSUM_IPV6,TXCSUM_IPV6>
        ether 00:0d:b9:4b:10:c0
        hwaddr 00:0d:b9:4b:10:c1
        media: Ethernet autoselect (1000baseT <full-duplex>)
        status: active
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
igb2: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=e120bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,WOL_MAGIC,VLAN_HWFILTER,RXCSUM_IPV6,TXCSUM_IPV6>
        ether 00:0d:b9:4b:10:c0
        hwaddr 00:0d:b9:4b:10:c2
        media: Ethernet autoselect (1000baseT <full-duplex>)
        status: active
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
        options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
        inet6 ::1 prefixlen 128
        inet6 fe80::1%lo0 prefixlen 64 scopeid 0x4
        inet 127.0.0.1 netmask 0xff000000
        groups: lo
        nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
pflog0: flags=0<> metric 0 mtu 33160
        groups: pflog
pfsync0: flags=0<> metric 0 mtu 1500
        syncpeer: 0.0.0.0 maxupd: 128 defer: off
        groups: pfsync
lagg0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=e120bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,WOL_MAGIC,VLAN_HWFILTER,RXCSUM_IPV6,TXCSUM_IPV6>
        ether 00:0d:b9:4b:10:c0
        laggproto lacp lagghash l2,l3,l4
        laggport: igb0 flags=1c<ACTIVE,COLLECTING,DISTRIBUTING>
        laggport: igb1 flags=1c<ACTIVE,COLLECTING,DISTRIBUTING>
        laggport: igb2 flags=1c<ACTIVE,COLLECTING,DISTRIBUTING>
        groups: lagg
        media: Ethernet autoselect
        status: active
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
switch-modem: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
        ether 02:27:54:9b:47:00
        id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
        maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
        root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
        member: epair0a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
                ifmaxaddr 0 port 14 priority 128 path cost 2000
        member: vlan10 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
                ifmaxaddr 0 port 11 priority 128 path cost 2000000
        groups: bridge
        nd6 options=9<PERFORMNUD,IFDISABLED>
switch-local: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
        ether 02:27:54:9b:47:01
        inet 192.168.0.11 netmask 0xffffff00 broadcast 192.168.0.255
        inet 192.168.0.20 netmask 0xffffffff broadcast 192.168.0.20
        inet 192.168.0.26 netmask 0xffffffff broadcast 192.168.0.26
        id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
        maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
        root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
        member: epair1a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
                ifmaxaddr 0 port 16 priority 128 path cost 2000
        member: vlan11 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
                ifmaxaddr 0 port 12 priority 128 path cost 2000000
        groups: bridge
        nd6 options=9<PERFORMNUD,IFDISABLED>
switch-pfsync: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
        ether 02:27:54:9b:47:02
        id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
        maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
        root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
        member: epair2a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
                ifmaxaddr 0 port 18 priority 128 path cost 2000
        member: vlan12 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
                ifmaxaddr 0 port 13 priority 128 path cost 2000000
        groups: bridge
        nd6 options=9<PERFORMNUD,IFDISABLED>
vlan10: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=200001<RXCSUM,RXCSUM_IPV6>
        ether 00:0d:b9:4b:10:c0
        groups: vlan
        vlan: 10 vlanpcp: 0 parent interface: lagg0
        media: Ethernet autoselect
        status: active
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
vlan11: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=200001<RXCSUM,RXCSUM_IPV6>
        ether 00:0d:b9:4b:10:c0
        groups: vlan
        vlan: 11 vlanpcp: 0 parent interface: lagg0
        media: Ethernet autoselect
        status: active
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
vlan12: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=200001<RXCSUM,RXCSUM_IPV6>
        ether 00:0d:b9:4b:10:c0
        groups: vlan
        vlan: 12 vlanpcp: 0 parent interface: lagg0
        media: Ethernet autoselect
        status: active
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
epair0a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
        description: vnet-fw1-modem
        options=8<VLAN_MTU>
        ether 02:1a:fe:59:ef:0a
        inet6 fe80::1a:feff:fe59:ef0a%epair0a prefixlen 64 scopeid 0xe
        groups: epair
        media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
        status: active
        nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
epair1a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
        description: vnet-fw1
        options=8<VLAN_MTU>
        ether 02:66:c9:0e:6e:0a
        inet6 fe80::66:c9ff:fe0e:6e0a%epair1a prefixlen 64 scopeid 0x10
        groups: epair
        media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
        status: active
        nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
epair2a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
        description: vnet-fw1-pfsync
        options=8<VLAN_MTU>
        ether 02:2e:49:e5:ae:0a
        inet6 fe80::2e:49ff:fee5:ae0a%epair2a prefixlen 64 scopeid 0x12
        groups: epair
        media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
        status: active
        nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>


### JAIL fw1 ifconfig ###

root@fw1:/ # ifconfig
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
        options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
        inet6 ::1 prefixlen 128
        inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
        inet 127.0.0.1 netmask 0xff000000
        groups: lo
        nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
pflog0: flags=141<UP,RUNNING,PROMISC> metric 0 mtu 33160
        groups: pflog
pfsync0: flags=41<UP,RUNNING> metric 0 mtu 1500
        pfsync: syncdev: pfsync syncpeer: 10.10.10.10 maxupd: 128 defer: off
        groups: pfsync
modem: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=8<VLAN_MTU>
        ether 02:1a:fe:59:ef:0b
        groups: epair
        media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
        status: active
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
local: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=8<VLAN_MTU>
        ether 02:66:c9:0e:6e:0b
        inet 192.168.0.252 netmask 0xffffff00 broadcast 192.168.0.255
        inet 192.168.0.254 netmask 0xffffffff broadcast 192.168.0.254 vhid 11
        groups: epair
        carp: MASTER vhid 11 advbase 1 advskew 50
        media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
        status: active
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
pfsync: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=8<VLAN_MTU>
        ether 02:2e:49:e5:ae:0b
        inet 10.10.10.9 netmask 0xfffffffc broadcast 10.10.10.11
        groups: epair
        media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
        status: active
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
tun0: flags=88d1<UP,POINTOPOINT,RUNNING,NOARP,SIMPLEX,MULTICAST> metric 0 mtu 1492
        inet 77.109.xx.xx --> 213.219.132.31 netmask 0xffffffff
        inet6 fe80::5c86:b03a:b54d:aca3%tun0 prefixlen 64 scopeid 0x7
        nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>