amirkdv/ Transparent Proxy to docker containers.md

## Transparent Proxy to docker containers.md

      
    Raw
  

               Transparent Proxy to docker containers.md
            
          
    This is an example of using Linux Kernel's
Transparent Proxy
to route all TCP traffic to docker containers without having to resort to
PROXY protocol
which is not supported by some applications
(e.g. sshd). To get the demo to work you only need vagrant installed:
git clone [this-gist] tproxy-demo
cd tproxy-demo
vagrant up
# follow instructions in the very last few lines of vagrant provisioner:

# tab #1
vagrant ssh -- sudo make -C /vagrant start_nc
# tab #2
vagrant ssh -- sudo make -C /vagrant start_haproxy
# tab #3
nc 192.168.33.10 9000 # tab 1 logs must show 192.168.33.1 (vagrant host) and not 127.0.0.1 (proxy IP)
Notes:

Vagrantfile hardcodes 192.168.33.10 for vagrant box IP address, if you
have another vagrant box with the same IP, change the IP and update haproxy.cfg
For this scheme to work, the HAProxy device must be positioned such that all
outgoing traffic of the proxied service (here netcat running inside a docker
container) passes through the proxy device as well. For this demo, this is
achievd by simply running HAProxy and docker host on the same machine. Otherwise
you must configure the docker hosts' default gateway.
others doing the same:

http://blog.loadbalancer.org/configure-haproxy-with-tproxy-kernel-for-full-transparent-proxy/
http://kb.snapt-ui.com/wp-content/uploads/2012/03/Snapt-HAProxy-TPROXY.pdf


## Dockerfile
FROM ubuntu:14.04
RUN apt-get update && apt-get install -y netcat-openbsd
CMD ["nc", "-vkl", "7000"]

## haproxy.cfg
global
  log     127.0.0.1  local0

defaults
  mode    tcp
  log     global
  option  tcplog
  retries 3
  maxconn 1000
  timeout connect 5s
  timeout client  15min
  timeout server  15min

listen tcp-in *:9000
  mode tcp
  log global
  # the following line is the magic, remove this and nc logs will show
  # proxy ip (127.0.0.1) as opposed to original client ip.
  source 0.0.0.0 usesrc clientip
  server srv-tcp 172.17.0.2:7000 # hardcodes docker container IP, might break

## Makefile
fix_locale:
	@echo '============================[fixing locale]============================'
	locale-gen en_CA.utf8
	update-locale LANG=en_CA.utf8

docker:
	@echo '==========================[installing docker]========================='
	apt-get update
	apt-get install -y apt-transport-https
	apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
	echo 'deb https://get.docker.io/ubuntu docker main' > /etc/apt/sources.list.d/docker.list
	apt-get update
	apt-get install -y lxc-docker

mods:
	@echo '==================[enabling xt_TPROXY and xt_socket]=================='
	find  /lib/modules/`uname -r` | grep -q xt_TPROXY
	find  /lib/modules/`uname -r` | grep -q xt_socket
	printf "xt_TPROXY\nxt_socket\n" | tee -a /etc/modules | xargs modprobe

net_config:
	@echo '================[configuring iptables rules for TPROXY]==============='
	iptables -t mangle -N DIVERT
	iptables -t mangle -A PREROUTING -p tcp -m socket -j DIVERT
	iptables -t mangle -A DIVERT -j MARK --set-mark 111
	iptables -t mangle -A DIVERT -j ACCEPT
	ip rule add fwmark 111 lookup 100
	ip route add local 0.0.0.0/0 dev lo table 100
	echo 1 > /proc/sys/net/ipv4/conf/all/forwarding

dependencies: docker mods net_config

build_nc:
	@echo '======================[building netcat container]====================='
	docker build -q -t amirkdv/nc .

build_haproxy:
	@echo '================[installing HAProxy w/ TPROXY support]================'
	apt-get update
	apt-get install -y build-essential make gcc
	wget http://www.haproxy.org/download/1.5/src/devel/haproxy-1.5-dev26.tar.gz
	tar -zxf haproxy-1.5-dev26.tar.gz
	make -C haproxy-1.5-dev26 TARGET=linux26 CPU=x86_64 USE_LINUX_TPROXY=1
	make -C haproxy-1.5-dev26 install target=linux26

build: fix_locale dependencies build_haproxy build_nc net_config

start_nc:
	@echo '==============[starting netcat container (port: 9876)]================'
	docker run -i -t --expose 7000 amirkdv/nc

start_haproxy:
	@echo '===================[starting haproxy in debug mode]==================='
	haproxy -f /vagrant/haproxy.cfg -d

demo:
	@echo '======================[transparent proxy demo]========================'
	@echo '1. in the first tab start the netcat container:'
	@echo '      vagrant ssh -- sudo make -C /vagrant start_nc'
	@echo '2. in a second tab start HAProxy:'
	@echo '      vagrant ssh -- sudo make -C /vagrant start_haproxy'
	@echo '3. in a third tab connect to the proxied netcat container and watch logs on tab 1 and 2:'
	@echo '      nc 192.168.33.10 9000'
	@echo '   you must see 192.168.33.1 (originating IP) in nc logs and not 172.17.42.1 (proxy ip)'

.PHONY: all

## Vagrantfile
# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure('2') do |config|
  config.vm.box = 'trusty64'
  config.vm.box_url = 'https://cloud-images.ubuntu.com/vagrant/trusty/current/trusty-server-cloudimg-amd64-vagrant-disk1.box'

  config.vm.hostname =  'transparent-proxy'
  config.vm.network :private_network, ip: '192.168.33.10'

  config.vm.provision 'shell', inline: 'make -C /vagrant build'
  config.vm.provision 'shell', inline: 'make -C /vagrant demo'

  # works around network slowness inside vagrant, see
  # https://github.com/mitchellh/vagrant/issues/1807
  config.vm.provider :virtualbox do |vb|
    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
    vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
  end
end
	FROM ubuntu:14.04
	RUN apt-get update && apt-get install -y netcat-openbsd
	CMD ["nc", "-vkl", "7000"]
	global
	log 127.0.0.1 local0

	defaults
	mode tcp
	log global
	option tcplog
	retries 3
	maxconn 1000
	timeout connect 5s
	timeout client 15min
	timeout server 15min

	listen tcp-in *:9000
	mode tcp
	log global
	# the following line is the magic, remove this and nc logs will show
	# proxy ip (127.0.0.1) as opposed to original client ip.
	source 0.0.0.0 usesrc clientip
	server srv-tcp 172.17.0.2:7000 # hardcodes docker container IP, might break
	fix_locale:
	@echo '============================[fixing locale]============================'
	locale-gen en_CA.utf8
	update-locale LANG=en_CA.utf8

	docker:
	@echo '==========================[installing docker]========================='
	apt-get update
	apt-get install -y apt-transport-https
	apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
	echo 'deb https://get.docker.io/ubuntu docker main' > /etc/apt/sources.list.d/docker.list
	apt-get update
	apt-get install -y lxc-docker

	mods:
	@echo '==================[enabling xt_TPROXY and xt_socket]=================='
	find /lib/modules/`uname -r` \| grep -q xt_TPROXY
	find /lib/modules/`uname -r` \| grep -q xt_socket
	printf "xt_TPROXY\nxt_socket\n" \| tee -a /etc/modules \| xargs modprobe

	net_config:
	@echo '================[configuring iptables rules for TPROXY]==============='
	iptables -t mangle -N DIVERT
	iptables -t mangle -A PREROUTING -p tcp -m socket -j DIVERT
	iptables -t mangle -A DIVERT -j MARK --set-mark 111
	iptables -t mangle -A DIVERT -j ACCEPT
	ip rule add fwmark 111 lookup 100
	ip route add local 0.0.0.0/0 dev lo table 100
	echo 1 > /proc/sys/net/ipv4/conf/all/forwarding

	dependencies: docker mods net_config

	build_nc:
	@echo '======================[building netcat container]====================='
	docker build -q -t amirkdv/nc .

	build_haproxy:
	@echo '================[installing HAProxy w/ TPROXY support]================'
	apt-get update
	apt-get install -y build-essential make gcc
	wget http://www.haproxy.org/download/1.5/src/devel/haproxy-1.5-dev26.tar.gz
	tar -zxf haproxy-1.5-dev26.tar.gz
	make -C haproxy-1.5-dev26 TARGET=linux26 CPU=x86_64 USE_LINUX_TPROXY=1
	make -C haproxy-1.5-dev26 install target=linux26

	build: fix_locale dependencies build_haproxy build_nc net_config

	start_nc:
	@echo '==============[starting netcat container (port: 9876)]================'
	docker run -i -t --expose 7000 amirkdv/nc

	start_haproxy:
	@echo '===================[starting haproxy in debug mode]==================='
	haproxy -f /vagrant/haproxy.cfg -d

	demo:
	@echo '======================[transparent proxy demo]========================'
	@echo '1. in the first tab start the netcat container:'
	@echo ' vagrant ssh -- sudo make -C /vagrant start_nc'
	@echo '2. in a second tab start HAProxy:'
	@echo ' vagrant ssh -- sudo make -C /vagrant start_haproxy'
	@echo '3. in a third tab connect to the proxied netcat container and watch logs on tab 1 and 2:'
	@echo ' nc 192.168.33.10 9000'
	@echo ' you must see 192.168.33.1 (originating IP) in nc logs and not 172.17.42.1 (proxy ip)'

	.PHONY: all
	# -- mode: ruby --
	# vi: set ft=ruby :

	Vagrant.configure('2') do \|config\|
	config.vm.box = 'trusty64'
	config.vm.box_url = 'https://cloud-images.ubuntu.com/vagrant/trusty/current/trusty-server-cloudimg-amd64-vagrant-disk1.box'

	config.vm.hostname = 'transparent-proxy'
	config.vm.network :private_network, ip: '192.168.33.10'

	config.vm.provision 'shell', inline: 'make -C /vagrant build'
	config.vm.provision 'shell', inline: 'make -C /vagrant demo'

	# works around network slowness inside vagrant, see
	# https://github.com/mitchellh/vagrant/issues/1807
	config.vm.provider :virtualbox do \|vb\|
	vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
	vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
	end
	end