Reproducer for stale tcp sessions

stale_tcp.go

package main

// Build and run as root since this code uses iptables to emulate a network outage.

// Make sure rules in iptables are cleared:
// sudo iptables -D INPUT -j DROP -p tcp --destination-port 2379; sudo iptables -D INPUT -j DROP -p tcp --source-port 2379

import (
	"context"
	"log"
	"os/exec"
	"time"

	"github.com/coreos/etcd/clientv3"
)

func main() {
	cli, err := clientv3.New(clientv3.Config{
		Endpoints:   []string{"localhost:2379"},
		DialTimeout: 5 * time.Second,
	})
	if err != nil {
		panic(err)
	}
	defer cli.Close()

	go func() {
		for {
			log.Println("++ Putting")
			ctx, _ := context.WithTimeout(context.Background(), time.Second)
			_, err := cli.Put(ctx, "/test", "somedata")
			if err != nil {
				log.Printf("Put error: %s\n", err)
			}
			time.Sleep(time.Second * 2)
		}
	}()

	go func() {
		log.Println("Starting watch")
		watchCh := cli.Watch(context.Background(), "/test")
		for ev := range watchCh {
			if ev.Err() != nil {
				log.Printf("Watch error: %s\n", ev.Err())
				continue
			}
			log.Printf("++ Watch: %+v", ev.Events)
		}
	}()

	cmd := exec.Command("bash", "-c", "netstat -nap | grep ':2379'")
	out, err := cmd.Output()
	if err != nil {
		log.Panicf("netstat failed", err)
	}
	log.Printf("netstat: %s\n", out)

	log.Println("Start dropping packets in 5 seconds")
	time.Sleep(time.Second * 5)

	log.Println("Dropping packets")
	cmd = exec.Command("bash", "-c", "iptables -A INPUT -j DROP -p tcp --destination-port 2379; iptables -A INPUT -j DROP -p tcp --source-port 2379")
	err = cmd.Run()
	if err != nil {
		log.Panicf("iptables failed", err)
	}
	cmd = exec.Command("bash", "-c", "netstat -nap | grep ':2379'")
	out, err = cmd.Output()
	if err != nil {
		log.Panicf("netstat failed", err)
	}
	log.Printf("netstat: %s\n", out)

	log.Println("Waiting for 10 seconds")
	time.Sleep(time.Second * 10)

	log.Println("Restoring connectivity")
	cmd = exec.Command("bash", "-c", "iptables -D INPUT -j DROP -p tcp --destination-port 2379; iptables -D INPUT -j DROP -p tcp --source-port 2379")
	err = cmd.Run()
	if err != nil {
		log.Panicf("iptables failed", err)
	}
	cmd = exec.Command("bash", "-c", "netstat -nap | grep ':2379'")
	out, err = cmd.Output()
	if err != nil {
		log.Panicf("netstat failed", err)
	}
	log.Printf("netstat: %s\n", out)

	log.Println("Waiting for 10 seconds")
	time.Sleep(time.Second * 10)

	cmd = exec.Command("bash", "-c", "netstat -nap | grep ':2379'")
	out, err = cmd.Output()
	if err != nil {
		log.Panicf("netstat failed", err)
	}
	log.Printf("netstat: %s\n", out)
}