Skip to content

Instantly share code, notes, and snippets.

@alistairncoles
Created March 21, 2016 19:44
Show Gist options
  • Save alistairncoles/cba6a7953748d054f5df to your computer and use it in GitHub Desktop.
Save alistairncoles/cba6a7953748d054f5df to your computer and use it in GitHub Desktop.
Monitoring swift container sync traffic
#!/bin/bash
# could be smarter and read these ports from the server conf files...
OBJ_SERVER_PORTS="6010 6020 6030 6040"
PROXY_SERVER_PORT=8080
setup_rules() {
# $1 should be -A to set, -D to unset
OP=$1
# the order we set the rules dictates the order they are displayed.
# bytes from obj servers...
for PORT in $OBJ_SERVER_PORTS
do
iptables $OP INPUT -p tcp --sport $PORT
done
# bytes to proxy server ...
iptables $OP INPUT -p tcp --dport $PROXY_SERVER_PORT
# bytes to object servers...
for PORT in $OBJ_SERVER_PORTS
do
iptables $OP INPUT -p tcp --dport $PORT
done
}
set() {
setup_rules -A
}
unset() {
setup_rules -D
}
show() {
iptables -nvxL INPUT
}
reset() {
iptables -Z INPUT
}
cmd=$0
if [[ -z $1 ]];
then
echo "$cmd [set|unset|reset|show]"
exit
fi
$1
echo "$1 rules on ports $PROXY_SERVER_PORT $OBJ_SERVER_PORTS"
@alistairncoles
Copy link
Author

Reviewing https://review.openstack.org/#/c/270961/

On master, commit 7cc2c78
=========================

inspect traffic between servers using container-sync.
Using a 3x replica policy

# set up single direction sync relationship...

$ swift post c1 -H 'X-Container-Sync-To: //saio/saio_endpoint/AUTH_test/c2'
$ swift post c1 -H 'X-Container-Sync-To: //saio/saio_endpoint/AUTH_test/c2'
$ swift post c1 -H 'X-Container-Sync-Key: foo'
$ swift post c2 -H 'X-Container-Sync-Key: foo'
$ 

# setup iptables rules to monitor port traffic stats...

$ sudo anc-ip-stats.bash reset
reset rules on ports 8080 6010 6020 6030 6040
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 25 packets, 1412 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040

# upload file, 4MB goes to proxy, 4MB to each object server...

$ swift upload c1 4MB_file
4MB_file
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 1041 packets, 16851467 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
       5      409            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
      58     3379            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
      52     3067            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
      57     3327            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     131  4202139            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
       5      675            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     137  4202471            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     132  4202211            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     129  4202055            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040

# reset traffic stats, run container sync on container server 3 (which happens to handle the object in first phase of sync)...
# 4MB from one object server, 4MB to proxy server, 4MB to each obj server

$ sudo anc-ip-stats.bash reset
reset rules on ports 8080 6010 6020 6030 6040
$ swift-init once container-sync -c 3
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/3.conf)
$ swift list c2
4MB_file
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 1660 packets, 22148462 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
       5      480            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
     114   532277            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
     116   532381            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
     223  4208067            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     189  4206979            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
       5      599            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     173  4207731            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     174  4207783            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     249  4211695            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040

# run container-sync *twice* on each of other container servers, which will process the object on their second pass...
# (except container-server 2 which does not have this container)
# on second pass, each container sync process reads 4MB from an object server and sends 4MB to proxy server,
# but the proxy does not send data on to object servers (100 continue supported internally to swift)

# container server 1...

$ swift-init once container-sync -c 1
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/1.conf)
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 2003 packets, 22189999 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
       5      480            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
     144   535391            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
     146   535495            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
     253  4211181            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     224  4210527            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
       5      599            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     203  4212645            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     204  4212697            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     279  4216609            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040
$ swift-init once container-sync -c 1
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/1.conf)
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 2546 packets, 32058775 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
      10      960            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
     170  1258268            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
     274  4737084            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
     278  4933741            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     334  8411088            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
      10     1198            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     217  4214450            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     267  4217050            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     293  4218414            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040

# container server 2... (does not have container)

$ swift-init once container-sync -c 2
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/2.conf)
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 2592 packets, 32060967 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
      10      960            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
     170  1258268            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
     274  4737084            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
     278  4933741            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     334  8411088            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
      10     1198            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     217  4214450            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     267  4217050            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     293  4218414            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040
$ swift-init once container-sync -c 2
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/2.conf)
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 2634 packets, 32062927 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
      10      960            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
     170  1258268            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
     274  4737084            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
     278  4933741            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     334  8411088            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
      10     1198            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     217  4214450            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     267  4217050            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     293  4218414            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040

# container server 4...

$ swift-init once container-sync -c 4
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/4.conf)
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 3032 packets, 32115018 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
      10      960            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
     200  1261382            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
     304  4740198            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
     308  4936855            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     369  8414636            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
      10     1198            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     247  4219364            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     297  4221964            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     323  4223328            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040
$ swift-init once container-sync -c 4
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/4.conf)
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 3797 packets, 42016899 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
      15     1440            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
     255  1987056            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
     364  5466397            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
     460  9141246            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     500 12618017            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
      15     1798            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     291  4226084            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     341  4228684            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     400  4231764            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040

# and back to container server 3...

$ swift-init once container-sync -c 3
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/3.conf)
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 4322 packets, 52014976 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
      20     1920            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
     282  2840686            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
     498  9668510            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
     485  9863806            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     590 16817538            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
      20     2397            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     306  4227941            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     398  4232725            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     414  4233569            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040

# sanity check, run all sync daemons again, no further data movement...

$ swift-init once container-sync
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/1.conf)
Running container-sync once...(/etc/swift/container-server/2.conf)
Running container-sync once...(/etc/swift/container-server/3.conf)
Running container-sync once...(/etc/swift/container-server/4.conf)
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 4380 packets, 52017756 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
      20     1920            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
     282  2840686            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
     498  9668510            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
     485  9863806            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     590 16817538            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
      20     2397            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     306  4227941            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     398  4232725            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     414  4233569            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040


# On master:   total data sent to proxy server is 16MB == 4 x object size
#              total data read from source object servers is approx 22MB
$ 



On review branch commit 054d4e6d
================================


$ swift upload c1 4MB_file
4MB_file
$ swift list c1 --lh
4.0M 2016-03-21 15:38:39 application/octet-stream 4MB_file
4.0M
$ swift list c2 --lh
4.0M 2016-03-21 14:50:49 application/octet-stream 4MB_file
4.0M
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 1747 packets, 16926640 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
      43     2450            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
      52     3264            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
      42     2398            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     194  4208124            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     133  4201848            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     138  4202523            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     132  4201796            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040

# run container sync on container server 3, which actually handles the object in first sync phase...
# 4MB read from an object server, 4MB put to proxy server, 4MB written to each obejct server.

$ sudo anc-ip-stats.bash reset
reset rules on ports 8080 6010 6020 6030 6040
$ swift-init once container-sync -c 3
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/3.conf)
$ swift list c2 --lh
4.0M 2016-03-21 15:38:39 application/octet-stream 4MB_file
4.0M
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 1446 packets, 22312608 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
       5      480            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
     183  4204445            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
      87   529319            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
      94   726690            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     160  4204027            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
       5      600            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     207  4206211            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     144  4202870            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     148  4203541            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040

# now run all container sync daemons repeatedly - no further object data traffic between servers...

$ swift-init once container-sync
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/1.conf)
Running container-sync once...(/etc/swift/container-server/2.conf)
Running container-sync once...(/etc/swift/container-server/3.conf)
Running container-sync once...(/etc/swift/container-server/4.conf)
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 1569 packets, 22320915 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
       5      480            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
     188  4205051            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
      87   529319            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
      94   726690            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     166  4204623            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
       5      600            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     212  4206934            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     144  4202870            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     148  4203541            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040
$ swift-init once container-sync
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/1.conf)
Running container-sync once...(/etc/swift/container-server/2.conf)
Running container-sync once...(/etc/swift/container-server/3.conf)
Running container-sync once...(/etc/swift/container-server/4.conf)
$ 
$ 
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 1730 packets, 22335528 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
       5      480            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
     198  4206263            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
      87   529319            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
      94   726690            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     178  4205815            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
       5      600            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     222  4208380            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     144  4202870            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     148  4203541            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040
$ swift-init once container-sync
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/1.conf)
Running container-sync once...(/etc/swift/container-server/2.conf)
Running container-sync once...(/etc/swift/container-server/3.conf)
Running container-sync once...(/etc/swift/container-server/4.conf)
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 1795 packets, 22338408 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
       5      480            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
     198  4206263            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
      87   529319            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
      94   726690            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     178  4205815            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
       5      600            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     222  4208380            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     144  4202870            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     148  4203541            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040


# Sanity check that object *will* be sync'd during second phase of sync daemon if missing...

$ swift upload c1 4MB_file
4MB_file
$ swift list c2 --lh

# run sync daemon on container server 1, object is not sync'd in first phase...

$ sudo anc-ip-stats.bash reset
reset rules on ports 8080 6010 6020 6030 6040
$ swift-init once container-sync -c 1
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/1.conf)
$ swift list c2 --lh
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 175 packets, 14903 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
      12      998            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
       0        0            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040

# Run daemon again on same server, this time object gets sync'd...

$ swift-init once container-sync -c 1
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/1.conf)
$ swift list c2 --lh
4.0M 2016-03-21 15:47:48 application/octet-stream 4MB_file
4.0M
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 1722 packets, 22333993 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
      10      889            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
      89   726007            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
      89   529611            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
     201  4205504            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     189  4205909            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
      10     1323            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     149  4203593            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     147  4203489            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     225  4207545            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040

# run daemon a few more times on all servers, no further object data traffic...

$ swift-init once container-sync
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/1.conf)
Running container-sync once...(/etc/swift/container-server/2.conf)
Running container-sync once...(/etc/swift/container-server/3.conf)
Running container-sync once...(/etc/swift/container-server/4.conf)
$ swift-init once container-sync
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/1.conf)
Running container-sync once...(/etc/swift/container-server/2.conf)
Running container-sync once...(/etc/swift/container-server/3.conf)
Running container-sync once...(/etc/swift/container-server/4.conf)
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 1939 packets, 22352337 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
      10      889            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
      94   726613            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
      94   530217            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
     206  4206110            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     207  4207697            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
      10     1323            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
     154  4204316            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     152  4204212            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     230  4208268            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040

# 
# We may still see unnecessary PUT traffic when remote object is inconsistent...
#

$ swift upload c1 4MB_file
4MB_file
$ swift list c1 --lh
4.0M 2016-03-21 16:00:07 application/octet-stream 4MB_file
4.0M

# kill a primary obj server, run sync daemon, sync is successful (2 PUTs to primary plus one to handoff)

$ swift-init stop object-server -c 2
Signal object-server  pid: 8722  signal: 15
object-server (8722) appears to have stopped
$ sudo anc-ip-stats.bash reset
reset rules on ports 8080 6010 6020 6030 6040
$ swift-init once container-sync -c 3
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/3.conf)
$ swift list c2 --lh
4.0M 2016-03-21 16:00:07 application/octet-stream 4MB_file
4.0M
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 1363 packets, 21523913 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
      64     3762            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
       2       80            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
      76   463264            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
     162  4203634            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     140  4202987            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
     140  4202661            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
       2      120            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     144  4202869            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     205  4206504            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040

# restart primary, stop other 2 primaries, so any HEAD request will now get old timestamp...

$ swift-init start object-server -c 2
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Starting object-server...(/etc/swift/object-server/2.conf)
$ swift-init stop object-server -c 3
Signal object-server  pid: 8742  signal: 15
object-server (8742) appears to have stopped
$ swift-init stop object-server -c 4
Signal object-server  pid: 8780  signal: 15
object-server (8780) appears to have stopped

$ swift stat c2 4MB_file 
       Account: AUTH_test
     Container: c2
        Object: 4MB_file
  Content Type: application/octet-stream
Content Length: 4194304
 Last Modified: Mon, 21 Mar 2016 15:47:49 GMT
          ETag: b5cfa9d6c8febd618f91ac2843d50a1c
    Meta Mtime: 1449839351.588577
 Accept-Ranges: bytes
   X-Timestamp: 1458575268.22501
    X-Trans-Id: txb704b89c78aa482ab2428-0056f01b38

# run container sync (second phase on same server), sync will fail because the remote PUT to handoff will 409...

$ swift-init once container-sync -c 3
WARNING: Unable to modify file descriptor limit.  Running as non-root?
Running container-sync once...(/etc/swift/container-server/3.conf)
$ swift stat c2 4MB_file 
       Account: AUTH_test
     Container: c2
        Object: 4MB_file
  Content Type: application/octet-stream
Content Length: 4194304
 Last Modified: Mon, 21 Mar 2016 15:47:49 GMT
          ETag: b5cfa9d6c8febd618f91ac2843d50a1c
    Meta Mtime: 1449839351.588577
 Accept-Ranges: bytes
   X-Timestamp: 1458575268.22501
    X-Trans-Id: txa632ee3d76a24bcca6c99-0056f01baa
$ sudo anc-ip-stats.bash show
Chain INPUT (policy ACCEPT 2646 packets, 30012939 bytes)
    pkts      bytes target     prot opt in     out     source               destination         
      73     4733            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6010
     146  4203528            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6020
      81   463464            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6030
     166  4203794            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp spt:6040
     308  8408174            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8080
     150  4204259            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6010
      93     7211            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6020
     149  4203169            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6030
     209  4206744            tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:6040
show rules on ports 8080 6010 6020 6030 6040

# Note: container sync does not report this as a failed sync, because the remote proxy returns a 202 when backends 409:

Mar 21 16:04:06 anc-vm-10 container-sync: Since Mon Mar 21 16:04:06 2016: 2 synced [0 deletes, 1 puts], 0 skipped, 0 failed
Mar 21 16:04:06 anc-vm-10 container-sync: Container sync "once" mode completed: 0.12s
Mar 21 16:04:06 anc-vm-10 container-sync: Exited

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment