Skip to content

Instantly share code, notes, and snippets.

@yudai
Created January 24, 2018 04:44
Show Gist options
  • Save yudai/a5e1269ef3d484217b10415af6dd08c4 to your computer and use it in GitHub Desktop.
Save yudai/a5e1269ef3d484217b10415af6dd08c4 to your computer and use it in GitHub Desktop.
Reproducer for stale tcp sessions
package main
// Build and run as root since this code uses iptables to emulate a network outage.
// Make sure rules in iptables are cleared:
// sudo iptables -D INPUT -j DROP -p tcp --destination-port 2379; sudo iptables -D INPUT -j DROP -p tcp --source-port 2379
import (
"context"
"log"
"os/exec"
"time"
"github.com/coreos/etcd/clientv3"
)
func main() {
cli, err := clientv3.New(clientv3.Config{
Endpoints: []string{"localhost:2379"},
DialTimeout: 5 * time.Second,
})
if err != nil {
panic(err)
}
defer cli.Close()
go func() {
for {
log.Println("++ Putting")
ctx, _ := context.WithTimeout(context.Background(), time.Second)
_, err := cli.Put(ctx, "/test", "somedata")
if err != nil {
log.Printf("Put error: %s\n", err)
}
time.Sleep(time.Second * 2)
}
}()
go func() {
log.Println("Starting watch")
watchCh := cli.Watch(context.Background(), "/test")
for ev := range watchCh {
if ev.Err() != nil {
log.Printf("Watch error: %s\n", ev.Err())
continue
}
log.Printf("++ Watch: %+v", ev.Events)
}
}()
cmd := exec.Command("bash", "-c", "netstat -nap | grep ':2379'")
out, err := cmd.Output()
if err != nil {
log.Panicf("netstat failed", err)
}
log.Printf("netstat: %s\n", out)
log.Println("Start dropping packets in 5 seconds")
time.Sleep(time.Second * 5)
log.Println("Dropping packets")
cmd = exec.Command("bash", "-c", "iptables -A INPUT -j DROP -p tcp --destination-port 2379; iptables -A INPUT -j DROP -p tcp --source-port 2379")
err = cmd.Run()
if err != nil {
log.Panicf("iptables failed", err)
}
cmd = exec.Command("bash", "-c", "netstat -nap | grep ':2379'")
out, err = cmd.Output()
if err != nil {
log.Panicf("netstat failed", err)
}
log.Printf("netstat: %s\n", out)
log.Println("Waiting for 10 seconds")
time.Sleep(time.Second * 10)
log.Println("Restoring connectivity")
cmd = exec.Command("bash", "-c", "iptables -D INPUT -j DROP -p tcp --destination-port 2379; iptables -D INPUT -j DROP -p tcp --source-port 2379")
err = cmd.Run()
if err != nil {
log.Panicf("iptables failed", err)
}
cmd = exec.Command("bash", "-c", "netstat -nap | grep ':2379'")
out, err = cmd.Output()
if err != nil {
log.Panicf("netstat failed", err)
}
log.Printf("netstat: %s\n", out)
log.Println("Waiting for 10 seconds")
time.Sleep(time.Second * 10)
cmd = exec.Command("bash", "-c", "netstat -nap | grep ':2379'")
out, err = cmd.Output()
if err != nil {
log.Panicf("netstat failed", err)
}
log.Printf("netstat: %s\n", out)
}
@yudai
Copy link
Author

yudai commented Jan 24, 2018

The output would look like (etcd binary is renamed to etcdd in this example):

$ sudo ./stale_tcp                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             2018/01/23 20:47:59 ++ Putting                                                                                                                                                                                                                                                                                                                                                                                                               2018/01/23 20:47:59 Starting watch                                                                                                                                                                                                                                                                                                                                                                                                           2018/01/23 20:47:59 ++ Watch: [0xc42025e000]                                                                                                                                                                                                                                                                                                                                                                                                 2018/01/23 20:47:59 netstat: tcp        0      0 127.0.0.1:2379          0.0.0.0:*               LISTEN      27878/etcdd                                                                                                                                                                                                                                                                                                                     tcp        0     17 127.0.0.1:2379          127.0.0.1:33272         ESTABLISHED 27878/etcdd                                                                                                                                                                                                                                                                                                                                                  tcp        0      0 127.0.0.1:2379          127.0.0.1:33236         ESTABLISHED 27878/etcdd                                                                                                                                                                                                                                                                                                                                                  tcp        0      0 127.0.0.1:33236         127.0.0.1:2379          ESTABLISHED 27878/etcdd                                                                                                                                                                                                                                                                                                                                                  tcp        0      0 127.0.0.1:33272         127.0.0.1:2379          ESTABLISHED 28443/stale_tcp
2018/01/23 20:47:59 Start dropping packets in 5 seconds
2018/01/23 20:48:01 ++ Putting
2018/01/23 20:48:01 ++ Watch: [0xc4201c42c0]
2018/01/23 20:48:03 ++ Putting
2018/01/23 20:48:03 ++ Watch: [0xc420154fa0]
2018/01/23 20:48:04 Dropping packets
2018/01/23 20:48:04 netstat: tcp        0      0 127.0.0.1:2379          0.0.0.0:*               LISTEN      27878/etcdd
tcp        0      0 127.0.0.1:2379          127.0.0.1:33272         ESTABLISHED 27878/etcdd
tcp        0      0 127.0.0.1:2379          127.0.0.1:33236         ESTABLISHED 27878/etcdd                                                                                                                                                                                                                                                                                                                                                  tcp        0      0 127.0.0.1:33236         127.0.0.1:2379          ESTABLISHED 27878/etcdd                                                                                                                                                                                                                                                                                                                                                  tcp        0      0 127.0.0.1:33272         127.0.0.1:2379          ESTABLISHED 28443/stale_tcp                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           2018/01/23 20:48:04 Waiting for 10 seconds                                                                                                                                                                                                                                                                                                                                                                                                   2018/01/23 20:48:05 ++ Putting                                                                                                                                                                                                                                                                                                                                                                                                               2018/01/23 20:48:06 Put error: context deadline exceeded                                                                                                                                                                                                                                                                                                                                                                                     2018/01/23 20:48:08 ++ Putting                                                                                                                                                                                                                                                                                                                                                                                                               2018/01/23 20:48:09 Put error: context deadline exceeded                                                                                                                                                                                                                                                                                                                                                                                     2018/01/23 20:48:11 ++ Putting
2018/01/23 20:48:12 Put error: context deadline exceeded
2018/01/23 20:48:14 ++ Putting
2018/01/23 20:48:14 Restoring connectivity
2018/01/23 20:48:14 netstat: tcp        0      0 127.0.0.1:2379          0.0.0.0:*               LISTEN      27878/etcdd
tcp        0      1 127.0.0.1:33298         127.0.0.1:2379          SYN_SENT    28443/stale_tcp
tcp        0      1 127.0.0.1:33282         127.0.0.1:2379          SYN_SENT    28443/stale_tcp                                                                                                                                                                                                                                                                                                                                              tcp        0      0 127.0.0.1:2379          127.0.0.1:33272         ESTABLISHED 27878/etcdd                                                                                                                                                                                                                                                                                                                                                  tcp        0      0 127.0.0.1:2379          127.0.0.1:33236         ESTABLISHED 27878/etcdd                                                                                                                                                                                                                                                                                                                                                  tcp        0      0 127.0.0.1:33236         127.0.0.1:2379          ESTABLISHED 27878/etcdd                                                                                                                                                                                                                                                                                                                                                  tcp        0      1 127.0.0.1:33290         127.0.0.1:2379          SYN_SENT    28443/stale_tcp
tcp        0     68 127.0.0.1:33272         127.0.0.1:2379          ESTABLISHED 28443/stale_tcp
2018/01/23 20:48:14 Waiting for 10 seconds
2018/01/23 20:48:15 ++ Watch: [0xc420155300]
2018/01/23 20:48:17 ++ Putting
2018/01/23 20:48:17 ++ Watch: [0xc420155420]
2018/01/23 20:48:19 ++ Putting
2018/01/23 20:48:19 ++ Watch: [0xc42025e540]
2018/01/23 20:48:21 ++ Putting
2018/01/23 20:48:21 ++ Watch: [0xc4201c4440]
2018/01/23 20:48:23 ++ Putting
2018/01/23 20:48:23 ++ Watch: [0xc42022e0e0]
2018/01/23 20:48:24 netstat: tcp        0      0 127.0.0.1:2379          0.0.0.0:*               LISTEN      27878/etcdd
tcp        0      0 127.0.0.1:33298         127.0.0.1:2379          ESTABLISHED 28443/stale_tcp
tcp        0      0 127.0.0.1:2379          127.0.0.1:33272         ESTABLISHED 27878/etcdd
tcp        0      0 127.0.0.1:2379          127.0.0.1:33236         ESTABLISHED 27878/etcdd
tcp        0      0 127.0.0.1:33236         127.0.0.1:2379          ESTABLISHED 27878/etcdd
tcp        0      0 127.0.0.1:33272         127.0.0.1:2379          ESTABLISHED 28443/stale_tcp
tcp        0      0 127.0.0.1:2379          127.0.0.1:33298         ESTABLISHED 27878/etcdd

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment