Transparent Proxy to docker containers

Transparent Proxy to docker containers.md

This is an example of using Linux Kernel's Transparent Proxy to route all TCP traffic to docker containers without having to resort to PROXY protocol which is not supported by some applications (e.g. sshd). To get the demo to work you only need vagrant installed:

git clone [this-gist] tproxy-demo
cd tproxy-demo
vagrant up
# follow instructions in the very last few lines of vagrant provisioner:

# tab #1
vagrant ssh -- sudo make -C /vagrant start_nc
# tab #2
vagrant ssh -- sudo make -C /vagrant start_haproxy
# tab #3
nc 192.168.33.10 9000 # tab 1 logs must show 192.168.33.1 (vagrant host) and not 127.0.0.1 (proxy IP)

Notes:

• Vagrantfile hardcodes 192.168.33.10 for vagrant box IP address, if you have another vagrant box with the same IP, change the IP and update haproxy.cfg
• For this scheme to work, the HAProxy device must be positioned such that all outgoing traffic of the proxied service (here netcat running inside a docker container) passes through the proxy device as well. For this demo, this is achievd by simply running HAProxy and docker host on the same machine. Otherwise you must configure the docker hosts' default gateway.
• others doing the same:
  • http://blog.loadbalancer.org/configure-haproxy-with-tproxy-kernel-for-full-transparent-proxy/
  • http://kb.snapt-ui.com/wp-content/uploads/2012/03/Snapt-HAProxy-TPROXY.pdf

Dockerfile

FROM ubuntu:14.04
RUN apt-get update && apt-get install -y netcat-openbsd
CMD ["nc", "-vkl", "7000"]

haproxy.cfg

global
  log     127.0.0.1  local0

defaults
  mode    tcp
  log     global
  option  tcplog
  retries 3
  maxconn 1000
  timeout connect 5s
  timeout client  15min
  timeout server  15min

listen tcp-in *:9000
  mode tcp
  log global
  # the following line is the magic, remove this and nc logs will show
  # proxy ip (127.0.0.1) as opposed to original client ip.
  source 0.0.0.0 usesrc clientip
  server srv-tcp 172.17.0.2:7000 # hardcodes docker container IP, might break

Makefile

fix_locale:
	@echo '============================[fixing locale]============================'
	locale-gen en_CA.utf8
	update-locale LANG=en_CA.utf8

docker:
	@echo '==========================[installing docker]========================='
	apt-get update
	apt-get install -y apt-transport-https
	apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
	echo 'deb https://get.docker.io/ubuntu docker main' > /etc/apt/sources.list.d/docker.list
	apt-get update
	apt-get install -y lxc-docker

mods:
	@echo '==================[enabling xt_TPROXY and xt_socket]=================='
	find  /lib/modules/`uname -r` | grep -q xt_TPROXY
	find  /lib/modules/`uname -r` | grep -q xt_socket
	printf "xt_TPROXY\nxt_socket\n" | tee -a /etc/modules | xargs modprobe

net_config:
	@echo '================[configuring iptables rules for TPROXY]==============='
	iptables -t mangle -N DIVERT
	iptables -t mangle -A PREROUTING -p tcp -m socket -j DIVERT
	iptables -t mangle -A DIVERT -j MARK --set-mark 111
	iptables -t mangle -A DIVERT -j ACCEPT
	ip rule add fwmark 111 lookup 100
	ip route add local 0.0.0.0/0 dev lo table 100
	echo 1 > /proc/sys/net/ipv4/conf/all/forwarding

dependencies: docker mods net_config

build_nc:
	@echo '======================[building netcat container]====================='
	docker build -q -t amirkdv/nc .

build_haproxy:
	@echo '================[installing HAProxy w/ TPROXY support]================'
	apt-get update
	apt-get install -y build-essential make gcc
	wget http://www.haproxy.org/download/1.5/src/devel/haproxy-1.5-dev26.tar.gz
	tar -zxf haproxy-1.5-dev26.tar.gz
	make -C haproxy-1.5-dev26 TARGET=linux26 CPU=x86_64 USE_LINUX_TPROXY=1
	make -C haproxy-1.5-dev26 install target=linux26

build: fix_locale dependencies build_haproxy build_nc net_config

start_nc:
	@echo '==============[starting netcat container (port: 9876)]================'
	docker run -i -t --expose 7000 amirkdv/nc

start_haproxy:
	@echo '===================[starting haproxy in debug mode]==================='
	haproxy -f /vagrant/haproxy.cfg -d

demo:
	@echo '======================[transparent proxy demo]========================'
	@echo '1. in the first tab start the netcat container:'
	@echo '      vagrant ssh -- sudo make -C /vagrant start_nc'
	@echo '2. in a second tab start HAProxy:'
	@echo '      vagrant ssh -- sudo make -C /vagrant start_haproxy'
	@echo '3. in a third tab connect to the proxied netcat container and watch logs on tab 1 and 2:'
	@echo '      nc 192.168.33.10 9000'
	@echo '   you must see 192.168.33.1 (originating IP) in nc logs and not 172.17.42.1 (proxy ip)'

.PHONY: all

Vagrantfile

# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure('2') do |config|
  config.vm.box = 'trusty64'
  config.vm.box_url = 'https://cloud-images.ubuntu.com/vagrant/trusty/current/trusty-server-cloudimg-amd64-vagrant-disk1.box'

  config.vm.hostname =  'transparent-proxy'
  config.vm.network :private_network, ip: '192.168.33.10'

  config.vm.provision 'shell', inline: 'make -C /vagrant build'
  config.vm.provision 'shell', inline: 'make -C /vagrant demo'

  # works around network slowness inside vagrant, see
  # https://github.com/mitchellh/vagrant/issues/1807
  config.vm.provider :virtualbox do |vb|
    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
    vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
  end
end

@amirkdv thank you for this guide!