AWS VPC VPN StrongSwan Virtual Tunnel Interface (VTI)

bgpd.conf

#@ /etc/quagga/bgpd.conf (Centos & Ubuntu)

hostname <Local OS hostname>
password <Any random phrase>
enable password <Any random phrase>
!
log file /var/log/quagga/bgpd
!debug bgp events
!debug bgp zebra
debug bgp updates
!
router bgp <Your Customer Gateway ASN>
  bgp router-id <Any integer number - smaller means higher priority routes>
  network <Your internal LAN subnet - e.g. 10.130.0.0/16>
  !network 169.254.x.x/32
  !network 169.254.x.x/32
  !
  ! aws tunnel #1 neighbour
  neighbor <Your VGW1 Inside IP> remote-as 17493
  !
  ! aws tunnel #2 neighbour
  neighbor <Your VGW2 Inside IP> remote-as 17493
  !
  ! Uncomment the line below if you prefer to use 'Connection B' as your backup (Connection A will # be used as your primary for all traffic). By default if you do not uncomment the next lines, traffic can #be sent and received down both of your connections at any time (asymmetric routing).
  !neighbor <Your VGW2 Inside IP> route-map RM_LOWER_PRIORITY out
!
route-map RM_LOWER_PRIORITY permit 10
  set as-path prepend <Your Customer Gateway ASN> <Your Customer Gateway ASN> <Your Customer Gateway ASN>
!
line vty


# LEGEND
# https://s3-us-west-2.amazonaws.com/youtubetutorials/racoon_config.txt
#
# <Your Customer Gateway ASN> is the local autonomous system (Customer Gateway ASN)
# 17493 is the remote autonomous system of AWS (Virtual Private  Gateway ASN)
#
# <Your VGW1 Inside IP> is the 169.x address on the REMOTE side of the first peer. (Neighbor IP Address)
# <Your VGW2 Inside IP> is the 169.x address on the REMOTE side of the second peer. (Neighbor IP Address)
#
# <Your internal LAN subnet - e.g. 10.0.0.0/16> is the local private subnet/LAN (Private Network Subnet)
# 169.254.x.x/32 is the 169.x address on LOCAL side of the first peer. Use /32
# 169.254.x.x/32 is the 169.x address on the LOCAL side of the second peer. Use /32

ipsec-vti.sh

#@ /etc/strongswan/ipsec-vti.sh (Centos) or /etc/strongswan.d/ipsec-vti.sh (Ubuntu)

#!/bin/bash
# AWS VPC Hardware VPN Strongswan updown Script

# Usage Instructions:
# Add "install_routes = no" to /etc/strongswan/strongswan.d/charon.conf or /etc/strongswan.d/charon.conf
# Add "install_virtual_ip = no" to /etc/strongswan/strongswan.d/charon.conf or /etc/strongswan.d/charon.conf
# For Ubuntu: Add "leftupdown=/etc/strongswan.d/ipsec-vti.sh" to /etc/ipsec.conf
# For RHEL/Centos: Add "leftupdown=/etc/strongswan/ipsec-vti.sh" to /etc/strongswan/ipsec.conf
# For RHEL/Centos 6 and below: git clone git://git.kernel.org/pub/scm/linux/kernel/git/shemminger/iproute2.git && cd iproute2 && make && cp ./ip/ip /usr/local/sbin/ip

# Adjust the below according to the Generic Gateway Configuration file provided to you by AWS.
# Sample: http://docs.aws.amazon.com/AmazonVPC/latest/NetworkAdminGuide/GenericConfig.html

IP=$(which ip)
IPTABLES=$(which iptables)

PLUTO_MARK_OUT_ARR=(${PLUTO_MARK_OUT//// })
PLUTO_MARK_IN_ARR=(${PLUTO_MARK_IN//// })
case "$PLUTO_CONNECTION" in
  AWS-VPC-GW1)
    VTI_INTERFACE=vti1
    VTI_LOCALADDR=<Your CGW1 Inside IP 169.254.x.x/30>
    VTI_REMOTEADDR=<Your VGW1 Inside IP 169.254.x.x/30>
    ;;
  AWS-VPC-GW2)
    VTI_INTERFACE=vti2
    VTI_LOCALADDR=<Your CGW2 Inside IP 169.254.x.x/30>
    VTI_REMOTEADDR=<Your VGW2 Inside IP 169.254.x.x/30>
    ;;
esac

case "${PLUTO_VERB}" in
    up-client)
        #$IP tunnel add ${VTI_INTERFACE} mode vti local ${PLUTO_ME} remote ${PLUTO_PEER} okey ${PLUTO_MARK_OUT_ARR[0]} ikey ${PLUTO_MARK_IN_ARR[0]}
        $IP link add ${VTI_INTERFACE} type vti local ${PLUTO_ME} remote ${PLUTO_PEER} okey ${PLUTO_MARK_OUT_ARR[0]} ikey ${PLUTO_MARK_IN_ARR[0]}
        sysctl -w net.ipv4.conf.${VTI_INTERFACE}.disable_policy=1
        sysctl -w net.ipv4.conf.${VTI_INTERFACE}.rp_filter=2 || sysctl -w net.ipv4.conf.${VTI_INTERFACE}.rp_filter=0
        $IP addr add ${VTI_LOCALADDR} remote ${VTI_REMOTEADDR} dev ${VTI_INTERFACE}
        $IP link set ${VTI_INTERFACE} up mtu 1436
	$IPTABLES -t mangle -I FORWARD -o ${VTI_INTERFACE} -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu
        $IPTABLES -t mangle -I INPUT -p esp -s ${PLUTO_PEER} -d ${PLUTO_ME} -j MARK --set-xmark ${PLUTO_MARK_IN}
        $IP route flush table 220
        #/etc/init.d/bgpd reload || /etc/init.d/quagga force-reload bgpd
        ;;
    down-client)
        #$IP tunnel del ${VTI_INTERFACE}
        $IP link del ${VTI_INTERFACE}
	$IPTABLES -t mangle -D FORWARD -o ${VTI_INTERFACE} -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu
        $IPTABLES -t mangle -D INPUT -p esp -s ${PLUTO_PEER} -d ${PLUTO_ME} -j MARK --set-xmark ${PLUTO_MARK_IN}
        ;;
esac

# Enable IPv4 forwarding
sysctl -w net.ipv4.ip_forward=1
sysctl -w net.ipv4.conf.eth1.disable_xfrm=1
sysctl -w net.ipv4.conf.eth1.disable_policy=1

# References:
# http://docs.aws.amazon.com/AmazonVPC/latest/NetworkAdminGuide/Introduction.html
# http://end.re/2015-01-06_vti-tunnel-interface-with-strongswan.html
# https://www-01.ibm.com/support/knowledgecenter/#!/SST55W_4.3.0/liaca/liaca_cfg_ipsec_vti.html

ipsec.conf

#@ /etc/strongswan/ipsec.conf (Centos) or /etc/ipsec.conf (Ubuntu)

# ipsec.conf - strongSwan IPsec configuration file

# basic configuration

config setup
       charondebug="cfg 2, ike 3"
#      strictcrlpolicy=yes
#      uniqueids = no

# Add connections here.

# Sample VPN connections

#conn sample-self-signed
#      leftsubnet=10.1.0.0/16
#      leftcert=selfCert.der
#      leftsendcert=never
#      right=192.168.0.2
#      rightsubnet=10.2.0.0/16
#      rightcert=peerCert.der
#      auto=start

#conn sample-with-ca-cert
#      leftsubnet=10.1.0.0/16
#      leftcert=myCert.pem
#      right=192.168.0.2
#      rightsubnet=10.2.0.0/16
#      rightid="C=CH, O=Linux strongSwan CN=peer name"
#      auto=start


# Usage Instructions:
# Adjust the below according to the Generic Gateway Configuration file provided to you by AWS.
# Sample: http://docs.aws.amazon.com/AmazonVPC/latest/NetworkAdminGuide/GenericConfig.html

conn %default
	# Authentication Method    : Pre-Shared Key
	#authby=psk
	leftauth=psk
	rightauth=psk
	# Encryption Algorithm     : aes-128-cbc
	# Authentication Algorithm : sha1
	# Perfect Forward Secrecy  : Diffie-Hellman Group 2
	ike=aes256-sha256-modp2048s256,aes128-sha1-modp1024!
	# Lifetime                 : 28800 seconds
	ikelifetime=28800s
	# Phase 1 Negotiation Mode : main
	aggressive=no
	# Protocol                 : esp
	# Encryption Algorithm     : aes-128-cbc
	# Authentication Algorithm : hmac-sha1-96
	# Perfect Forward Secrecy  : Diffie-Hellman Group 2
	esp=aes128-sha256-modp2048s256,aes128-sha1-modp1024!
	# Lifetime                 : 3600 seconds
	lifetime=3600s
	# Mode                     : tunnel
	type=tunnel
	# DPD Interval             : 10
	dpddelay=10s
	# DPD Retries              : 3
	dpdtimeout=30s
	# Tuning Parameters for AWS Virtual Private Gateway:
	keyexchange=ikev1
	#keyingtries=%forever
	rekey=yes
	reauth=no
	dpdaction=restart
	closeaction=restart
	left=%defaultroute
	leftsubnet=0.0.0.0/0,::/0
	rightsubnet=0.0.0.0/0,::/0
	leftupdown=/etc/strongswan/ipsec-vti.sh
	installpolicy=yes
	compress=no
	mobike=no

conn AWS-VPC-GW1
	# Customer Gateway:        : <Your Strongswan-CGW Public IP>
	left=<Your Strongswan-CGW Public IP>
	# Virtual Private Gateway  : <Your VGW1 Outside IP>
	right=<Your VGW1 Outside IP>
	auto=start
	mark=100
	#reqid=1

conn AWS-VPC-GW2
	# Customer Gateway:        : <Your Strongswan-CGW Public IP>
	left=<Your Strongswan-CGW Public IP>
	# Virtual Private Gateway  : <Your VGW2 Outside IP>
	right=<Your VGW2 Outside IP>
	auto=start
	mark=200
	#reqid=2

# References:
# http://docs.aws.amazon.com/AmazonVPC/latest/NetworkAdminGuide/Introduction.html
# http://end.re/2015-01-06_vti-tunnel-interface-with-strongswan.html
# https://www-01.ibm.com/support/knowledgecenter/#!/SST55W_4.3.0/liaca/liaca_cfg_ipsec_vti.html
# https://aravindkrishnaswamy.wordpress.com/tag/multiple-vpn-tunnels-with-strongswan/
# https://aravindkrishnaswamy.wordpress.com/2014/11/26/site-to-site-vpn-between-openvpn-and-aws/
# http://www.mynameistoby.com/blog/2015/01/21/setting-up-strongswan-on-centos-6-to-connect-to-your-amazon-vpc-vpn/
# https://wiki.strongswan.org/projects/strongswan/wiki/ConnSection
# https://wiki.strongswan.org/projects/strongswan/wiki/IKEv2CipherSuites

ipsec.secrets

#@ /etc/strongswan/ipsec.secrets (Centos) or /etc/ipsec.secrets (Ubuntu)

<Your Strongswan-CGW Public IP> <Your VGW1 Outside IP> : PSK "<Replace with VGW1 secret phrase provided by AWS>"
<Your Strongswan-CGW Public IP> <Your VGW2 Outside IP> : PSK "<Replace with VGW2 secret phrase provided by AWS>"

iptables

#@ /etc/sysconfig/iptables (Centos) or /etc/iptables/rules.v4 (Ubuntu with iptables-persistent package)

# Generated by iptables-save v1.4.7
*filter
:INPUT DROP [1:60]
:FORWARD DROP [0:0]
:OUTPUT ACCEPT [21:2888]
-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
-A INPUT -p tcp -m tcp --tcp-flags FIN,SYN,RST,PSH,ACK,URG NONE -j DROP
-A INPUT -p tcp -m tcp ! --tcp-flags FIN,SYN,RST,ACK SYN -m state --state NEW -j DROP
-A INPUT -p tcp -m tcp --tcp-flags FIN,SYN,RST,PSH,ACK,URG FIN,SYN,RST,PSH,ACK,URG -j DROP
-A INPUT -i lo -j ACCEPT
-A INPUT -p tcp -m tcp --dport 22 -j ACCEPT
-A INPUT -i eth0 -p esp -j ACCEPT
-A INPUT -i eth0 -p udp -m udp --sport 500 --dport 500 -j ACCEPT
-A INPUT -i eth0 -p udp -m udp --sport 4500 --dport 4500 -j ACCEPT
-A INPUT -i vti+ -p tcp -m tcp --dport 179 -j ACCEPT
-A INPUT -i eth+ -p icmp -m icmp --icmp-type 8 -j ACCEPT
-A FORWARD -i eth1 -p icmp -m icmp --icmp-type 8 -j ACCEPT
-A FORWARD -p icmp -m icmp --icmp-type 0 -m state --state RELATED,ESTABLISHED -j ACCEPT
-A OUTPUT -p icmp -m icmp --icmp-type 0 -m state --state RELATED,ESTABLISHED -j ACCEPT
COMMIT
# Completed

zebra.conf

#@ /etc/quagga/zebra.conf (Centos & Ubuntu)

hostname <Local OS hostname>
password <Any random phrase>
enable password <Any random phrase>
!
! Configure interfaces
interface lo
! Change preferred source ip address of received routes
route-map RM_SET_SRC permit 10
  set src <Your host ip-address on Your internal LAN subnet interface - e.g. 10.130.0.5>
ip protocol bgp route-map RM_SET_SRC
!
line vty

It was extremely useful. We used this information and built very stable VTI router, posted here - https://github.com/eis-group-opensource/vti-router . The changes was:

• most important, CentOS7 native kernel is not compatible, you must update to non native kernel 4
• we keep configs for each vti interface in separate files, 3 for each .init for IP config (used by script), .conf for ipsec conf, , and separate PSK file.
• AWS description parser added
• Interaction with quagga added, BGP added / removed, static to the far end is now via quagga not in sysconfig file
• 4 states for each vti interface - available, disabled, enabled, removed
• all changes are restart - less and interruption - less (so we can add / delete vti interfaces without disrupting others)
• health script added which re-connect failed IPSEC-s
• all tested under heavy load, with AWS and Azure, and 10 - 16 interfaces / 1 VTI router (total of 10+ routers are in use).
• management scripts added - to make initial setup, enter VTI information manually or by AWS file parsing, add/delete/enable/disable tunnels (on the fly, without any restarts, check status).

In case you are trying this from a Amazon Linux AMI, e.g. connecting from one VPC to another, StrongSwan cannot be yum:d, since it depends on libtspi (secure hardware storage for RSA keys) which is in base RHEL, but not in Amazon's base. So you have to:

yum-config-manager --enable epel
yum install yum-utils
yumdownloader strongswan
rpm -i --nodeps strongswan-5.4.0-2.el6.x86_64.rpm 

With this and the config above, it is possible to get at least a static route setup (i.e. no Quagga) going with just ip route add 172.16.0.0 via 169.254.X.Y added to ipsec-vti.sh. Kudos to @heri16.

Also, the $IPTABLES -t mangle -I INPUT -p esp -s ${PLUTO_PEER} -d ${PLUTO_ME} -j MARK --set-xmark ${PLUTO_MARK_IN} assumes no NAT-T? Looking at my iptables -t mangle -n -L -v gives

Chain INPUT (policy ACCEPT 682 packets, 93160 bytes)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       esp  --  *      *       <VGW-IP>       172.31.10.6          MARK set 0x64

My setup is NAT-T:d, being inside another VPC. The script could be improved to handle this, according to StrongSwan 4.4 changelog

PLUTO_ESP_ENC environment variables can be used in a user-specific updown script to set marks on inbound ESP or ESP_IN_UDP packets.

The script could be extended to match such packets with -p udp --dport 4500 or something to that effect.

(Note also https://wiki.strongswan.org/issues/1497 which appears to be fixed in the epel 6 strongswan-5.4.0-2.el6.x86_64 that I am using.)

Why "$IP link set ${VTI_INTERFACE} up mtu 1436" ?
Linux EC2 instances use jumbo frames and interfaces are set to MTU 9001. I increased ${VTI_INTERFACE} to 8937, but traffic stopped.
Any suggestion?

@miguelaustro, AWS VPC VPN only supports MTUs of 1436 bytes.
In order to utilize jumbo frames you need support for jumbo frames across public internet, which won't happen. All frames will be fragmented to 1436 bytes each.

God bless you! You leveled me from zero to vpn master in a gist 👍

Other helpful notes for anything noob like me in matter of VPNs:

• Remember to enable the various processes to be started at boot
• Remember to make ipsec-vti.sh executable
• At least in my case, in bgpd.conf the router-id must be set to an IP address. It doesn't have to be an existent IP or a resolvable IP. It just needs to be an IP that bgpd will use as an identifier.
• Remember that after this setup your instance will be able to reach VPC IPs but it will still not be able to resolve private hostnames. You need to create a Route53 Inbound Endpoint and use resolvconf to setup these custom nameservers:
  • A tutorial explaining how to create Inbound endpoints: https://medium.com/searce/resolve-aws-rds-and-other-dns-names-on-vpn-networks-using-r53-resolvers-55ec68271270
  • A tutorial explaining how to add custom nameservers: https://www.tecmint.com/set-permanent-dns-nameservers-in-ubuntu-debian/

You're welcome @nicolaracco

Thanks for the nice pointers! @aprudnev @bittrance @kaaelhaa @nicolaracco

The above minimalist config has been used for many years with no issues. It's nice to see what @aprudnev has done with it.

I'm currently looking for a job after a misadventure with a Blockchain outfit, so if anyone has any job opening in any part of the world, please drop me a message at Twitter handle "heri16". Thanks!

What additional steps are required to make this vpn host act as a gateway for its lan?

In my case AWS network is 172.0.0.0/16 and on-prem LAN is 10.3.0.0/16. VPN Gateway is 10.3.128.164.
The VPN gateway is able to reach any host inside the VPC (e.g. it can ssh in EC2 instances).
On another host of the lan (10.3.128.160) I ran route add -net 172.0.0.0 netmask 255.255.0.0 gw 10.3.128.164 dev eth2. But if I try from this host to ssh into a VPC EC2 it cannot reach the instance.
Running tcpdump on VPN gateway shows this output:

09:41:39.924771 IP ip-10-3-128-160.eu-central-1.compute.internal.37058 > ip-172-0-4-64.eu-central-1.compute.internal.ssh: Flags [S], seq 3268810629, win 26880, options [mss 8960,sackOK,TS val 513820347 ecr 0,nop,wscale 7], length 0
09:41:48.116888 IP ip-10-3-128-160.eu-central-1.compute.internal.37058 > ip-172-0-4-64.eu-central-1.compute.internal.ssh: Flags [S], seq 3268810629, win 26880, options [mss 8960,sackOK,TS val 513828539 ecr 0,nop,wscale 7], length 0
09:42:04.245177 IP ip-10-3-128-160.eu-central-1.compute.internal.37058 > ip-172-0-4-64.eu-central-1.compute.internal.ssh: Flags [S], seq 3268810629, win 26880, options [mss 8960,sackOK,TS val 513844667 ecr 0,nop,wscale 7], length 0
09:42:37.525175 IP ip-10-3-128-160.eu-central-1.compute.internal.37058 > ip-172-0-4-64.eu-central-1.compute.internal.ssh: Flags [S], seq 3268810629, win 26880, options [mss 8960,sackOK,TS val 513877947 ecr 0,nop,wscale 7], length 0
09:42:42.645264 ARP, Request who-has ip-10-3-128-164.eu-central-1.compute.internal tell ip-10-3-128-160.eu-central-1.compute.internal, length 42
09:42:42.645298 ARP, Reply ip-10-3-128-164.eu-central-1.compute.internal is-at fa:16:3e:55:1e:35 (oui Unknown), length 28

Solution

I was just missing to configure iptables for allowing forwarding :)
By default it seems only ICMP is allowed for forwarding on eth1. So, to allow anything from private LAN to AWS should be something like:

iptables -A FORWARD -i eth1 -J ACCEPT
iptables -A FORWARD -o eth1 -J ACCEPT

1. Make sure that traffic from your locsl system reach VTI gateway. We must add unsymmetrucal traffic bypasd to our firewalls in some cases, or you add static routevon yiur system . 2. Make sure that VTI router receive bgp routing. Post here VTI -status So we can see it. 3.Make sure that AWS accept your bgp routing. At least, have something like network 10.3.0.0/16 in bgp section of quagga. Send me netstat -rn vtysh sh ip bgp 10.3.0.0 sh ip bgp sum

…

On Fri, Feb 21, 2020, 1:32 AM Nicola Racco ***@***.***> wrote: What additional steps are required to make this vpn host act as a gateway for its lan? In my case AWS network is 172.0.0.0/16 and on-prem LAN is 10.3.0.0/16. VPN Gateway is 10.3.128.164. The VPN gateway is able to reach any host inside the VPC (e.g. it can ssh in EC2 instances). On another host of the lan (10.3.128.160) I ran route add -net 172.0.0.0 netmask 255.255.0.0 gw 10.3.128.164 dev eth2. But if I try from this host to ssh into a VPC EC2 it cannot reach the instance. Running tcpdump on VPN gateway shows this output: 09:31:10.762344 IP ip-10-3-128-160.eu-central-1.compute.internal.37056 > ip-172-0-4-64.eu-central-1.compute.internal.ssh: Flags [S], seq 4069779690, win 26880, options [mss 8960,sackOK,TS val 513191180 ecr 0,nop,wscale 7], length 0 09:31:11.764843 IP ip-10-3-128-160.eu-central-1.compute.internal.37056 > ip-172-0-4-64.eu-central-1.compute.internal.ssh: Flags [S], seq 4069779690, win 26880, options [mss 8960,sackOK,TS val 513192183 ecr 0,nop,wscale 7], length 0 09:31:13.780911 IP ip-10-3-128-160.eu-central-1.compute.internal.37056 > ip-172-0-4-64.eu-central-1.compute.internal.ssh: Flags [S], seq 4069779690, win 26880, options [mss 8960,sackOK,TS val 513194199 ecr 0,nop,wscale 7], length 0 09:31:17.844864 IP ip-10-3-128-160.eu-central-1.compute.internal.37056 > ip-172-0-4-64.eu-central-1.compute.internal.ssh: Flags [S], seq 4069779690, win 26880, options [mss 8960,sackOK,TS val 513198263 ecr 0,nop,wscale 7], length 0 09:31:26.036817 IP ip-10-3-128-160.eu-central-1.compute.internal.37056 > ip-172-0-4-64.eu-central-1.compute.internal.ssh: Flags [S], seq 4069779690, win 26880, options [mss 8960,sackOK,TS val 513206455 ecr 0,nop,wscale 7], length 0 09:31:42.165372 IP ip-10-3-128-160.eu-central-1.compute.internal.37056 > ip-172-0-4-64.eu-central-1.compute.internal.ssh: Flags [S], seq 4069779690, win 26880, options [mss 8960,sackOK,TS val 513222583 ecr 0,nop,wscale 7], length 0 ***@***.***:31:47.285422 ARP, Request who-has ip-10-3-128-164.eu-central-1.compute.internal tell ip-10-3-128-160.eu-central-1.compute.internal, length 42 09:31:47.285455 ARP, Reply ip-10-3-128-164.eu-central-1.compute.internal is-at fa:16:3e:55:1e:35 (oui Unknown), length 28 But I have no idea how to interpret it — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub <https://gist.github.com/2f59d22d1d5980796bfb?email_source=notifications&email_token=AFQ2RZL5D2TFMBJK5VMQDXDRD6NTDA5CNFSM4KYQIE72YY3PNVWWK3TUL52HS4DFVNDWS43UINXW23LFNZ2KUY3PNVWWK3TUL5UWJTQAGCLQU#gistcomment-3184394>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AFQ2RZNM7S2KVGFMRUYDG6TRD6NTDANCNFSM4KYQIE7Q> .

hello, currently i'm trying to establish Strongswan to AWS VPC using BGP.

My first question: do the the MARKs and vti-sh script is needed ? Why it's not enough to have just vti tunnel created and route traffic like in GRE case ? Thanks

There is a lot of specifics, how it all works. For me, I created working system, it is used in more than 30 - 40 cases in our company, we have it all tuned for auto restart, never fail, and so on, and so, as maybe other configurations can work, I do not see a reason to investigate them - as I need working tunnel and not R&D research _what else can be done_. MARK is required to label traffic for the tunnel. As it works this way - interface add MARK and then IP level forward marked traffic into the IPSEC. Do not forget that IPSEC, in it's classic form, use IP addresses to decide which traffic to encrypt and wrap in IPSEC. GRE + IPSEC combination is DIFFERENT way to create encrypted tunnels. We do use it in our company for usual site/site traffic. But it contains some overhead (as IPSEC itself is tunnel) and so Cisco introduce special IPSEC type, VTI, which presents IPSEC to the routing layer as any interface (and they called it VTI interface). When we use strongswan, we simulate VTI by labeling traffic (on pseudo interface vti*) and they saying to IPSEC to encrypt and tunnel it by recognizing it by the lavel. IPSEC tunnel, in case of VTI, has security domain as 0.0.0.0/0_0.0.0.0/0 which means _encrypt any traffic_ (so it does not use IP to decide what to encrypt) and we add MARK as a label (so it encrypt only MARKED traffic). Alternative is POLICY BASED IPSEC, AZURE has it, but it is extremely inconvenient and difficult to manage thing. AWS choose do not implement it, as it can not be used in routing.

…

On Wed, Apr 28, 2021 at 2:05 AM Edvinas K ***@***.***> wrote: ***@***.**** commented on this gist. ------------------------------ hello, currently i'm trying to establish Strongswan to AWS VPC using BGP. My first question: do the the MARKs and vti-sh script is needed ? Why it's not enough to have just vti tunnel created and route traffic like in GRE case ? Thanks — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub <https://gist.github.com/2f59d22d1d5980796bfb#gistcomment-3722804>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AFQ2RZLMVD7KEQUNIKRLKZ3TK7FWHANCNFSM4KYQIE7Q> .

There is a lot of specifics, how it all works. For me, I created working system, it is used in more than 30 - 40 cases in our company, we have it all tuned for auto restart, never fail, and so on, and so, as maybe other configurations can work, I do not see a reason to investigate them - as I need working tunnel and not R&D research what else can be done. MARK is required to label traffic for the tunnel. As it works this way - interface add MARK and then IP level forward marked traffic into the IPSEC. Do not forget that IPSEC, in it's classic form, use IP addresses to decide which traffic to encrypt and wrap in IPSEC. GRE + IPSEC combination is DIFFERENT way to create encrypted tunnels. We do use it in our company for usual site/site traffic. But it contains some overhead (as IPSEC itself is tunnel) and so Cisco introduce special IPSEC type, VTI, which presents IPSEC to the routing layer as any interface (and they called it VTI interface). When we use strongswan, we simulate VTI by labeling traffic (on pseudo interface vti*) and they saying to IPSEC to encrypt and tunnel it by recognizing it by the lavel. IPSEC tunnel, in case of VTI, has security domain as 0.0.0.0/0_0.0.0.0/0 which means encrypt any traffic (so it does not use IP to decide what to encrypt) and we add MARK as a label (so it encrypt only MARKED traffic). Alternative is POLICY BASED IPSEC, AZURE has it, but it is extremely inconvenient and difficult to manage thing. AWS choose do not implement it, as it can not be used in routing.
…
On Wed, Apr 28, 2021 at 2:05 AM Edvinas K @.> wrote: @.* commented on this gist. ------------------------------ hello, currently i'm trying to establish Strongswan to AWS VPC using BGP. My first question: do the the MARKs and vti-sh script is needed ? Why it's not enough to have just vti tunnel created and route traffic like in GRE case ? Thanks — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub https://gist.github.com/2f59d22d1d5980796bfb#gistcomment-3722804, or unsubscribe https://github.com/notifications/unsubscribe-auth/AFQ2RZLMVD7KEQUNIKRLKZ3TK7FWHANCNFSM4KYQIE7Q .

thanks for such a quick answer. You know - i tried to do 'simple` VTI + Strongswan + FRR (BGP) configuration between two internal nodes.

    left=10.34.32.154
    leftsubnet=10.34.32.154/32
    rightsubnet=10.34.32.155/32
    leftfirewall=no
    ike=aes-sha1-modp1024
    esp=aes128gcm16-modp1024
    right=10.34.32.155
    type=tunnel
    authby=psk
    auto=start
    keyexchange=ikev2
    mobike=no
    reauth=no

4: ipip0@NONE: <POINTOPOINT,NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ipip 10.34.32.154 peer 10.34.32.155
    inet 169.1.1.1/30 scope global ipip0
       valid_lft forever preferred_lft forever

And it works. So i thought the same should work connecting to AWS VPC. But sadly something is not OK. So trying to understand where's the catch. Why it works OK between two internal nodes but not between Strongswan - and VPC.

You created policy based IPSEC with this security domain - leftsubnet= 10.34.32.154/32 rightsubnet=10.34.32.155/32 THis IPSEC do not support routing and, even if you had interface for it, did not create VTI interface. An idea of VTI was that IPSEC is for ANY traffic which comes into it,l and traffic is determined by routing not IPSEC policy (this is why they have MARK).

…

On Wed, Apr 28, 2021 at 2:54 AM Edvinas K ***@***.***> wrote: ***@***.**** commented on this gist. ------------------------------ There is a lot of specifics, how it all works. For me, I created working system, it is used in more than 30 - 40 cases in our company, we have it all tuned for auto restart, never fail, and so on, and so, as maybe other configurations can work, I do not see a reason to investigate them - as I need working tunnel and not R&D research *what else can be done*. MARK is required to label traffic for the tunnel. As it works this way - interface add MARK and then IP level forward marked traffic into the IPSEC. Do not forget that IPSEC, in it's classic form, use IP addresses to decide which traffic to encrypt and wrap in IPSEC. GRE + IPSEC combination is DIFFERENT way to create encrypted tunnels. We do use it in our company for usual site/site traffic. But it contains some overhead (as IPSEC itself is tunnel) and so Cisco introduce special IPSEC type, VTI, which presents IPSEC to the routing layer as any interface (and they called it VTI interface). When we use strongswan, we simulate VTI by labeling traffic (on pseudo interface vti*) and they saying to IPSEC to encrypt and tunnel it by recognizing it by the lavel. IPSEC tunnel, in case of VTI, has security domain as 0.0.0.0/0_0.0.0.0/0 which means *encrypt any traffic* (so it does not use IP to decide what to encrypt) and we add MARK as a label (so it encrypt only MARKED traffic). Alternative is POLICY BASED IPSEC, AZURE has it, but it is extremely inconvenient and difficult to manage thing. AWS choose do not implement it, as it can not be used in routing. … <#m_-976555760427766450_> On Wed, Apr 28, 2021 at 2:05 AM Edvinas K *@*.*> wrote: @.** commented on this gist. ------------------------------ hello, currently i'm trying to establish Strongswan to AWS VPC using BGP. My first question: do the the MARKs and vti-sh script is needed ? Why it's not enough to have just vti tunnel created and route traffic like in GRE case ? Thanks — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub https://gist.github.com/2f59d22d1d5980796bfb#gistcomment-3722804, or unsubscribe https://github.com/notifications/unsubscribe-auth/AFQ2RZLMVD7KEQUNIKRLKZ3TK7FWHANCNFSM4KYQIE7Q . thanks for such a quick answer. You know - i tried to do 'simple` VTI + Strongswan + FRR (BGP) configuration between two internal nodes. left=10.34.32.154 leftsubnet=10.34.32.154/32 rightsubnet=10.34.32.155/32 leftfirewall=no ike=aes-sha1-modp1024 esp=aes128gcm16-modp1024 right=10.34.32.155 type=tunnel authby=psk auto=start keyexchange=ikev2 mobike=no reauth=no 4: ***@***.***: <POINTOPOINT,NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN group default qlen 1000 link/ipip 10.34.32.154 peer 10.34.32.155 inet 169.1.1.1/30 scope global ipip0 valid_lft forever preferred_lft forever And it works. So i thought the same should work connecting to AWS VPC. But sadly something is not OK. So trying to understand where's the catch. Why it works OK between two internal nodes but not between Strongswan - and VPC. — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub <https://gist.github.com/2f59d22d1d5980796bfb#gistcomment-3722948>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AFQ2RZJKOVM5YW3HXKKGRZ3TK7LMHANCNFSM4KYQIE7Q> .

thanks, somehow i manage ,y configuration to work.

The only strange thing i see that when i look into mangle table. I see no packets which supposed to be marked MATCHED. But it's still works. Can you check how it's on your side ? Thanks

 pkts bytes target     prot opt in     out     source               destination         

Chain INPUT (policy ACCEPT 171M packets, 170G bytes)
 pkts bytes target     prot opt in     out     source               destination         
      0     0 MARK       esp  --  *      *       18.138.xx.63        xx.xx.4.251         MARK set 0x64
      0     0 MARK       esp  --  *      *       54.169.xx.249       xx.xx.4.251         MARK set 0xc8

Chain FORWARD (policy ACCEPT 83M packets, 108G bytes)
 pkts bytes target     prot opt in     out     source               destination         
64837 3889K TCPMSS     tcp  --  *      VTI_awssg1  0.0.0.0/0            0.0.0.0/0            tcp flags:0x06/0x02 TCPMSS clamp to PMTU
  807 48404 TCPMSS     tcp  --  *      VTI_awssg2  0.0.0.0/0            0.0.0.0/0            tcp flags:0x06/0x02 TCPMSS clamp to PMTU

Chain OUTPUT (policy ACCEPT 74M packets, 61G bytes)
 pkts bytes target     prot opt in     out     source               destination         

Chain POSTROUTING (policy ACCEPT 159M packets, 170G bytes)
 pkts bytes target     prot opt in     out     source               destination         
You have mail in /var/spool/mail/root```