Workaround for https://github.com/docker/docker/issues/13914

dockeriptablesfix.go

// This program is a workaround for
// https://github.com/docker/docker/issues/13914
//
// Sometimes docker fails to properly clean up DOCKER routing rules and then
// traffic to the new version of the container gets blackholed.  Running this
// scripts frequently as root should mitigate this.
package main

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/Sirupsen/logrus"
	docker "github.com/fsouza/go-dockerclient"
)

var iptablesPath string

func initIptablesPath() {
	var err error
	iptablesPath, err = exec.LookPath("iptables")
	if err != nil {
		logrus.WithError(err).Fatal("can't find iptables")
	}
}

func runIptables(args ...string) ([]byte, *logrus.Entry) {
	lr := logrus.WithFields(logrus.Fields{
		"command": iptablesPath,
		"args":    args,
	})
	lr.Info("running")

	c := exec.Command(iptablesPath, args...)

	var stdoutBuffer bytes.Buffer
	c.Stdout = &stdoutBuffer
	var stderrBuffer bytes.Buffer
	c.Stderr = &stderrBuffer

	err := c.Run()
	stdout := stdoutBuffer.Bytes()
	stderr := stderrBuffer.Bytes()

	if err != nil {
		return nil, lr.WithError(err).WithFields(logrus.Fields{
			"stdout": string(stdout),
			"stderr": string(stderr),
		})
	}
	return stdout, nil
}

type exposedPort struct {
	protocol string
	hostPort string
}
type destination string

type fullMapping struct {
	exposedPort
	destination
}

func readMappingsFromIptables() map[exposedPort][]destination {
	mappingsByExposed := make(map[exposedPort][]destination)

	list, errLog := runIptables("-t", "nat", "-S", "DOCKER")
	if errLog != nil {
		errLog.Fatal("can't list routes")
	}
	lines := bufio.NewScanner(bytes.NewReader(list))
	for lines.Scan() {
		line := lines.Text()
		tokens := strings.Split(line, " ")
		if len(tokens) == 0 {
			// Skip blank lines if any.
			continue
		}
		lr := logrus.WithFields(logrus.Fields{"line": line, "tokens": tokens})
		if !(len(tokens) == 15 &&
			tokens[0] == "-A" && tokens[1] == "DOCKER" &&
			tokens[2] == "!" && tokens[3] == "-i" && tokens[4] == "docker0" &&
			tokens[5] == "-p" &&
			tokens[6] == tokens[8] &&
			tokens[7] == "-m" &&
			tokens[9] == "--dport" &&
			tokens[11] == "-j" && tokens[12] == "DNAT" && tokens[13] == "--to-destination") {
			continue
		}
		exposed := exposedPort{
			protocol: tokens[6],
			hostPort: tokens[10],
		}
		dest := destination(tokens[14])
		if exposed.protocol != "tcp" && exposed.protocol != "udp" {
			lr.WithField("protocol", exposed.protocol).Fatal("unknown protocol")
		}
		mappingsByExposed[exposed] = append(mappingsByExposed[exposed], dest)
	}
	return mappingsByExposed
}

func readMappingsFromDocker(dockerClient *docker.Client) map[exposedPort]destination {
	mappings := make(map[exposedPort]destination)
	containers, err := dockerClient.ListContainers(docker.ListContainersOptions{})
	if err != nil {
		logrus.WithError(err).Fatal("could not list containers")
	}
	for _, container := range containers {
		if len(container.Ports) == 0 {
			continue
		}
		// Look up the container's IP (this is *not* container.Ports[i].IP, which is
		// the external binding interface).
		fullContainer, err := dockerClient.InspectContainer(container.ID)
		if err != nil {
			logrus.WithError(err).WithField("container", container.ID).Fatal("could not inspect container")
		}
		ip := fullContainer.NetworkSettings.IPAddress
		for _, port := range container.Ports {
			exposed := exposedPort{
				protocol: port.Type,
				hostPort: fmt.Sprintf("%d", port.PublicPort),
			}
			dest := destination(fmt.Sprintf("%s:%d", ip, port.PrivatePort))

			if oldDest, found := mappings[exposed]; found {
				// The point of this tool is to resolve duplicate mappings in
				// iptables. If we see duplicate mappings in docker itself... then
				// that's just weird.
				logrus.WithFields(logrus.Fields{
					"exposed": exposed,
					"newDest": dest,
					"oldDest": oldDest,
				}).Fatal("duplicate mapping found in Docker")
			}
			mappings[exposed] = dest
		}
	}
	return mappings
}

func main() {
	initIptablesPath()

	dockerClient, err := docker.NewClientFromEnv()
	if err != nil {
		logrus.WithError(err).Fatal("can't find docker")
	}

	realMappings := readMappingsFromDocker(dockerClient)
	mappingsByExposed := readMappingsFromIptables()

	var mappingsToDelete []fullMapping

	for exposed, destinations := range mappingsByExposed {
		// We only try to clean up actual duplicates, because otherwise it's hard to
		// tell the difference between "old mapping that failed to be deleted" and
		// "new mapping for a container being created right now".
		if len(destinations) < 2 {
			continue
		}
		lr := logrus.WithFields(logrus.Fields{
			"exposed":      exposed,
			"destinations": destinations,
		})
		lr.WithField("n", len(destinations)).Info("duplicate routing detected")

		// Figure out which mapping docker thinks we should use.
		correctDest, ok := realMappings[exposed]
		if !ok {
			lr.Warn("no real mapping found")
			continue
		}
		lr = lr.WithField("correctDestination", correctDest)

		foundCorrectDest := false
		for _, dest := range destinations {
			if dest == correctDest {
				foundCorrectDest = true
				break
			}
		}
		if !foundCorrectDest {
			lr.Warn("correct dest not found in mappings")
			continue
		}

		for _, dest := range destinations {
			if dest != correctDest {
				mappingsToDelete = append(mappingsToDelete, fullMapping{exposedPort: exposed, destination: dest})
			}
		}
	}

	failed := false
	for _, mapping := range mappingsToDelete {
		if !deleteMapping(mapping) {
			failed = true
		}
	}
	if failed {
		os.Exit(1)
	}
}

func deleteMapping(mapping fullMapping) bool {
	args := []string{
		"-t", "nat", "-D", "DOCKER", "!", "-i", "docker0", "-p", mapping.protocol, "-m", mapping.protocol,
		"--dport", mapping.hostPort, "-j", "DNAT", "--to-destination", string(mapping.destination),
	}

	_, errLog := runIptables(args...)
	if errLog != nil {
		// Not fatal! We can try others. We will make the process exit 1 though.
		errLog.Error("can't delete mapping")
		return false
	}
	return true
}