nate-ray/Bro_network_performance_notes.txt Secret

## Bro_network_performance_notes.txt
### HARDWARE ###
-----------------------------------------------------------------------------------------------
Intel Xeon E5405 Quad-Core @ 2.00GHZ
6GB RAM
Intel 82571EB GB Copper NIC
-----------------------------------------------------------------------------------------------


### DRIVER ###
-----------------------------------------------------------------------------------------------
#update to latest drivers - updated from e1000e 2.3.2-k -> e1000e 3.2.4.2-NAPI
-----------------------------------------------------------------------------------------------

### NIC SETTINGS ###
-----------------------------------------------------------------------------------------------
#changed MTU from 1500 -> MTU 8192

#turn off rx checksumming
ethtool -K eth0 rx off

# off tx checksumming
ethtool -K eth0 tx off

#turn off scatter-gather
ethtool -K eth0 sg off

#turn off tcp-segmentation-offload
ethtool -K eth0 tso off

#turn off generic-segmentation-offload
ethtool -K eth0 gso off

#turn off generic-receive-offload
ethtool -K eth0 gro off

# ethtool -g eth0
Ring parameters for eth0:
Pre-set maximums:
RX:             4096
RX Mini:        0
RX Jumbo:       0
TX:             4096
Current hardware settings:
RX:             4096
RX Mini:        0
RX Jumbo:       0
TX:             64

# ethtool -S eth0
NIC statistics:
     rx_packets: 1343275020
     tx_packets: 1220
     rx_bytes: 659555318969
     tx_bytes: 135698
     rx_broadcast: 111
     tx_broadcast: 0
     rx_multicast: 71939
     tx_multicast: 1220
     rx_errors: 0
     tx_errors: 0
     tx_dropped: 0
     multicast: 71939
     collisions: 0
     rx_length_errors: 0
     rx_over_errors: 0
     rx_crc_errors: 0
     rx_frame_errors: 0
     rx_no_buffer_count: 0
     rx_missed_errors: 0
     tx_aborted_errors: 0
     tx_carrier_errors: 0
     tx_fifo_errors: 0
     tx_heartbeat_errors: 0
     tx_window_errors: 0
     tx_abort_late_coll: 0
     tx_deferred_ok: 0
     tx_single_coll_ok: 0
     tx_multi_coll_ok: 0
     tx_timeout_count: 0
     tx_restart_queue: 0
     rx_long_length_errors: 0
     rx_short_length_errors: 0
     rx_align_errors: 0
     tx_tcp_seg_good: 0
     tx_tcp_seg_failed: 0
     rx_flow_control_xon: 0
     rx_flow_control_xoff: 0
     tx_flow_control_xon: 0
     tx_flow_control_xoff: 0
     rx_csum_offload_good: 0
     rx_csum_offload_errors: 0
     rx_header_split: 1112111622
     alloc_rx_buff_failed: 0
     tx_smbus: 0
     rx_smbus: 0
     dropped_smbus: 0
     rx_dma_failed: 0
     tx_dma_failed: 0
     rx_hwtstamp_cleared: 0
     uncorr_ecc_errors: 0
     corr_ecc_errors: 0

# ethtool -c eth0
Coalesce parameters for eth0:
Adaptive RX: off  TX: off
stats-block-usecs: 0
sample-interval: 0
pkt-rate-low: 0
pkt-rate-high: 0

rx-usecs: 18
rx-frames: 0
rx-usecs-irq: 0
rx-frames-irq: 0

tx-usecs: 0
tx-frames: 0
tx-usecs-irq: 0
tx-frames-irq: 0

rx-usecs-low: 0
rx-frame-low: 0
tx-usecs-low: 0
tx-frame-low: 0

rx-usecs-high: 0
rx-frame-high: 0
tx-usecs-high: 0
tx-frame-high: 0
-----------------------------------------------------------------------------------------------

### PF_RING SETTINGS ###
-----------------------------------------------------------------------------------------------
# cat /proc/net/pf_ring/info
PF_RING Version          : 6.1.1 (dev:6a976d02509188fe99294b4b0a4ff4b1442dada0)
Total rings              : 1

Standard (non DNA/ZC) Options
Ring slots               : 4096
Slot version             : 16
Capture TX               : Yes [RX+TX]
IP Defragment            : No
Socket Mode              : Standard
Total plugins            : 0
Cluster Fragment Queue   : 0
Cluster Fragment Discard : 0

# cat /proc/net/pf_ring/14332-eth0.199
Bound Device(s)    : eth0
Active             : 1
Breed              : Standard
Appl. Name         : <unknown>
Socket Mode        : RX+TX
Capture Direction  : RX+TX
Sampling Rate      : 1
IP Defragment      : No
BPF Filtering      : Enabled
# Sw Filt. Rules   : 0
# Hw Filt. Rules   : 0
Poll Pkt Watermark : 1
Num Poll Calls     : 0
Channel Id Mask    : 0xFFFFFFFFFFFFFFFF
Cluster Id         : 0
Slot Version       : 16 [6.1.1]
Min Num Slots      : 4096
Bucket Len         : 8192
Slot Len           : 8224 [bucket+header]
Tot Memory         : 33697792
Tot Packets        : 153298629
Tot Pkt Lost       : 60413245
Tot Insert         : 92885384
Tot Read           : 92829402
Insert Offset      : 3522336
Remove Offset      : 3537608
Num Free Slots     : 0
TX: Send Ok        : 0
TX: Send Errors    : 0
Reflect: Fwd Ok    : 0
Reflect: Fwd Errors: 0


# ./pfcount -i eth0 -e 1
=========================
Absolute Stats: [1984405 pkts rcvd][0 pkts dropped]
Total Pkts=1984405/Dropped=0.0 %
1'984'405 pkts - 1'380'155'753 bytes [60'054.02 pkt/sec - 334.14 Mbit/sec]
=========================
Actual Stats: 1815 pkts [36.55 ms][49'659.36 pps/0.25 Gbps]
=========================
-----------------------------------------------------------------------------------------------

### BRO ###
-----------------------------------------------------------------------------------------------
# running for about 1hr15min
# /opt/bro/bin/broctl netstats
 bro: 1441984428.266905 recvd=98991567 dropped=69439867 link=98991567

#/opt/bro/bin/broctl status
Name         Type       Host          Status    Pid    Peers  Started
bro          standalone localhost     running   14332  0      11 Sep 09:02:42

#/opt/bro/bin/broctl config
[BroControl] > config
bindir = /opt/bro/bin
bro = /opt/bro/bin/bro
bro-crashed = False
bro-expect-running = True
bro-host = localhost
bro-pid = 14332
bro-port = 47760
broargs =
brobase = /opt/bro
broctlconfigdir = /opt/bro/spool
broport = 47760
broscriptdir = /opt/bro/share/bro
broversion = 2.4-125
capstatspath = /opt/bro/bin/capstats
cfgdir = /opt/bro/etc
cflowaddress =
cflowpassword =
cflowuser =
commandtimeout = 60
commtimeout = 10
compresscmd = gzip -9
compressextension = gz
compresslogs = 1
configchksum = 03e4b525dc0f4a282352dcbaa9da8e99
confignodechksum = 91e7e9acab1862487f22d9f3c15ece02
croncmd =
cronenabled = True
debug = 0
debuglog = /opt/bro/spool/debug.log
disk-space-localhost-dev-cciss-c0d0p1 = 8.379852919
env_vars =
hash-broctlcfg = dfb8fa5dd143e7bee94abc6b8a1ce577
hash-nodecfg = 90abe783d4329f8e3ab4e2c76597a3f6
havenfs = 0
helperdir = /opt/bro/share/broctl/scripts/helpers
ipv6comm = 1
keeplogs =
lastpkts-bro = 247342.0
libdir = /opt/bro/lib
libdirinternal = /opt/bro/lib/broctl
localnetscfg = /opt/bro/etc/networks.cfg
lockfile = /opt/bro/spool/lock
logdir = /opt/bro/logs
logexpireinterval = 14
logrotationinterval = 3600
mailalarmsinterval = 86400
mailalarmsto = root@localhost
mailconnectionsummary = True
mailfrom = Big Brother <bro@cap01sndcmo>
mailhostupdown = True
mailreplyto =
mailsubjectprefix = [Bro]
mailto = root@localhost
makearchivename = /opt/bro/share/broctl/scripts/make-archive-name
memlimit = unlimited
mindiskspace = 5
nodecfg = /opt/bro/etc/node.cfg
os = linux
pfringclusterid = 21
pfringclustertype = 4-tuple
pfringfirstappinstance = 0
pin_command = taskset -c
plugindir = /opt/bro/lib/broctl/plugins
policydir = /opt/bro/share/bro
policydirsiteinstall = /opt/bro/spool/installed-scripts-do-not-touch/site
policydirsiteinstallauto = /opt/bro/spool/installed-scripts-do-not-touch/auto
postprocdir = /opt/bro/share/broctl/scripts/postprocessors
prefixes = local
savetraces = 0
scriptsdir = /opt/bro/share/broctl/scripts
sendmail = /usr/sbin/sendmail
sitepluginpath =
sitepolicymanager = local-manager.bro
sitepolicypath = /opt/bro/share/bro/site
sitepolicystandalone = local.bro
sitepolicyworker = local-worker.bro
spooldir = /opt/bro/spool
standalone = True
statefile = /opt/bro/spool/state.db
staticdir = /opt/bro/share/broctl
statsdir = /opt/bro/logs/stats
statslog = /opt/bro/spool/stats.log
statslogenable = True
statslogexpireinterval = 0
statuscmdshowall = True
stoptimeout = 60
test.enabled = False
test.foo = 1
time = /usr/bin/time
timefmt = %d %b %H:%M:%S
timemachinehost =
timemachineport = 47757/tcp
tmpdir = /opt/bro/spool/tmp
tmpexecdir = /opt/bro/spool/tmp
tracesummary = /opt/bro/bin/trace-summary
version = 1.4-28
zoneid =
-----------------------------------------------------------------------------------------------

### NIC BUFFER CONFIG ###
-----------------------------------------------------------------------------------------------
echo 'net.core.rmem.max=25165824' >> /etc/sysctl.conf
echo 'net.core.rmem.default=25165824' >> /etc/sysctl.conf
echo 'net.core.netdev_max_backlog=1000000' >> /etc/sysctl.conf
echo 'net.ipv4.tcp_rmem= 20480 174760 25165824' >> /etc/sysctl.conf
sudo sysctl -p
	### HARDWARE ###
	-----------------------------------------------------------------------------------------------
	Intel Xeon E5405 Quad-Core @ 2.00GHZ
	6GB RAM
	Intel 82571EB GB Copper NIC
	-----------------------------------------------------------------------------------------------


	### DRIVER ###
	-----------------------------------------------------------------------------------------------
	#update to latest drivers - updated from e1000e 2.3.2-k -> e1000e 3.2.4.2-NAPI
	-----------------------------------------------------------------------------------------------

	### NIC SETTINGS ###
	-----------------------------------------------------------------------------------------------
	#changed MTU from 1500 -> MTU 8192

	#turn off rx checksumming
	ethtool -K eth0 rx off

	# off tx checksumming
	ethtool -K eth0 tx off

	#turn off scatter-gather
	ethtool -K eth0 sg off

	#turn off tcp-segmentation-offload
	ethtool -K eth0 tso off

	#turn off generic-segmentation-offload
	ethtool -K eth0 gso off

	#turn off generic-receive-offload
	ethtool -K eth0 gro off

	# ethtool -g eth0
	Ring parameters for eth0:
	Pre-set maximums:
	RX: 4096
	RX Mini: 0
	RX Jumbo: 0
	TX: 4096
	Current hardware settings:
	RX: 4096
	RX Mini: 0
	RX Jumbo: 0
	TX: 64

	# ethtool -S eth0
	NIC statistics:
	rx_packets: 1343275020
	tx_packets: 1220
	rx_bytes: 659555318969
	tx_bytes: 135698
	rx_broadcast: 111
	tx_broadcast: 0
	rx_multicast: 71939
	tx_multicast: 1220
	rx_errors: 0
	tx_errors: 0
	tx_dropped: 0
	multicast: 71939
	collisions: 0
	rx_length_errors: 0
	rx_over_errors: 0
	rx_crc_errors: 0
	rx_frame_errors: 0
	rx_no_buffer_count: 0
	rx_missed_errors: 0
	tx_aborted_errors: 0
	tx_carrier_errors: 0
	tx_fifo_errors: 0
	tx_heartbeat_errors: 0
	tx_window_errors: 0
	tx_abort_late_coll: 0
	tx_deferred_ok: 0
	tx_single_coll_ok: 0
	tx_multi_coll_ok: 0
	tx_timeout_count: 0
	tx_restart_queue: 0
	rx_long_length_errors: 0
	rx_short_length_errors: 0
	rx_align_errors: 0
	tx_tcp_seg_good: 0
	tx_tcp_seg_failed: 0
	rx_flow_control_xon: 0
	rx_flow_control_xoff: 0
	tx_flow_control_xon: 0
	tx_flow_control_xoff: 0
	rx_csum_offload_good: 0
	rx_csum_offload_errors: 0
	rx_header_split: 1112111622
	alloc_rx_buff_failed: 0
	tx_smbus: 0
	rx_smbus: 0
	dropped_smbus: 0
	rx_dma_failed: 0
	tx_dma_failed: 0
	rx_hwtstamp_cleared: 0
	uncorr_ecc_errors: 0
	corr_ecc_errors: 0

	# ethtool -c eth0
	Coalesce parameters for eth0:
	Adaptive RX: off TX: off
	stats-block-usecs: 0
	sample-interval: 0
	pkt-rate-low: 0
	pkt-rate-high: 0

	rx-usecs: 18
	rx-frames: 0
	rx-usecs-irq: 0
	rx-frames-irq: 0

	tx-usecs: 0
	tx-frames: 0
	tx-usecs-irq: 0
	tx-frames-irq: 0

	rx-usecs-low: 0
	rx-frame-low: 0
	tx-usecs-low: 0
	tx-frame-low: 0

	rx-usecs-high: 0
	rx-frame-high: 0
	tx-usecs-high: 0
	tx-frame-high: 0
	-----------------------------------------------------------------------------------------------

	### PF_RING SETTINGS ###
	-----------------------------------------------------------------------------------------------
	# cat /proc/net/pf_ring/info
	PF_RING Version : 6.1.1 (dev:6a976d02509188fe99294b4b0a4ff4b1442dada0)
	Total rings : 1

	Standard (non DNA/ZC) Options
	Ring slots : 4096
	Slot version : 16
	Capture TX : Yes [RX+TX]
	IP Defragment : No
	Socket Mode : Standard
	Total plugins : 0
	Cluster Fragment Queue : 0
	Cluster Fragment Discard : 0

	# cat /proc/net/pf_ring/14332-eth0.199
	Bound Device(s) : eth0
	Active : 1
	Breed : Standard
	Appl. Name : <unknown>
	Socket Mode : RX+TX
	Capture Direction : RX+TX
	Sampling Rate : 1
	IP Defragment : No
	BPF Filtering : Enabled
	# Sw Filt. Rules : 0
	# Hw Filt. Rules : 0
	Poll Pkt Watermark : 1
	Num Poll Calls : 0
	Channel Id Mask : 0xFFFFFFFFFFFFFFFF
	Cluster Id : 0
	Slot Version : 16 [6.1.1]
	Min Num Slots : 4096
	Bucket Len : 8192
	Slot Len : 8224 [bucket+header]
	Tot Memory : 33697792
	Tot Packets : 153298629
	Tot Pkt Lost : 60413245
	Tot Insert : 92885384
	Tot Read : 92829402
	Insert Offset : 3522336
	Remove Offset : 3537608
	Num Free Slots : 0
	TX: Send Ok : 0
	TX: Send Errors : 0
	Reflect: Fwd Ok : 0
	Reflect: Fwd Errors: 0


	# ./pfcount -i eth0 -e 1
	=========================
	Absolute Stats: [1984405 pkts rcvd][0 pkts dropped]
	Total Pkts=1984405/Dropped=0.0 %
	1'984'405 pkts - 1'380'155'753 bytes [60'054.02 pkt/sec - 334.14 Mbit/sec]
	=========================
	Actual Stats: 1815 pkts [36.55 ms][49'659.36 pps/0.25 Gbps]
	=========================
	-----------------------------------------------------------------------------------------------

	### BRO ###
	-----------------------------------------------------------------------------------------------
	# running for about 1hr15min
	# /opt/bro/bin/broctl netstats
	bro: 1441984428.266905 recvd=98991567 dropped=69439867 link=98991567

	#/opt/bro/bin/broctl status
	Name Type Host Status Pid Peers Started
	bro standalone localhost running 14332 0 11 Sep 09:02:42

	#/opt/bro/bin/broctl config
	[BroControl] > config
	bindir = /opt/bro/bin
	bro = /opt/bro/bin/bro
	bro-crashed = False
	bro-expect-running = True
	bro-host = localhost
	bro-pid = 14332
	bro-port = 47760
	broargs =
	brobase = /opt/bro
	broctlconfigdir = /opt/bro/spool
	broport = 47760
	broscriptdir = /opt/bro/share/bro
	broversion = 2.4-125
	capstatspath = /opt/bro/bin/capstats
	cfgdir = /opt/bro/etc
	cflowaddress =
	cflowpassword =
	cflowuser =
	commandtimeout = 60
	commtimeout = 10
	compresscmd = gzip -9
	compressextension = gz
	compresslogs = 1
	configchksum = 03e4b525dc0f4a282352dcbaa9da8e99
	confignodechksum = 91e7e9acab1862487f22d9f3c15ece02
	croncmd =
	cronenabled = True
	debug = 0
	debuglog = /opt/bro/spool/debug.log
	disk-space-localhost-dev-cciss-c0d0p1 = 8.379852919
	env_vars =
	hash-broctlcfg = dfb8fa5dd143e7bee94abc6b8a1ce577
	hash-nodecfg = 90abe783d4329f8e3ab4e2c76597a3f6
	havenfs = 0
	helperdir = /opt/bro/share/broctl/scripts/helpers
	ipv6comm = 1
	keeplogs =
	lastpkts-bro = 247342.0
	libdir = /opt/bro/lib
	libdirinternal = /opt/bro/lib/broctl
	localnetscfg = /opt/bro/etc/networks.cfg
	lockfile = /opt/bro/spool/lock
	logdir = /opt/bro/logs
	logexpireinterval = 14
	logrotationinterval = 3600
	mailalarmsinterval = 86400
	mailalarmsto = root@localhost
	mailconnectionsummary = True
	mailfrom = Big Brother <bro@cap01sndcmo>
	mailhostupdown = True
	mailreplyto =
	mailsubjectprefix = [Bro]
	mailto = root@localhost
	makearchivename = /opt/bro/share/broctl/scripts/make-archive-name
	memlimit = unlimited
	mindiskspace = 5
	nodecfg = /opt/bro/etc/node.cfg
	os = linux
	pfringclusterid = 21
	pfringclustertype = 4-tuple
	pfringfirstappinstance = 0
	pin_command = taskset -c
	plugindir = /opt/bro/lib/broctl/plugins
	policydir = /opt/bro/share/bro
	policydirsiteinstall = /opt/bro/spool/installed-scripts-do-not-touch/site
	policydirsiteinstallauto = /opt/bro/spool/installed-scripts-do-not-touch/auto
	postprocdir = /opt/bro/share/broctl/scripts/postprocessors
	prefixes = local
	savetraces = 0
	scriptsdir = /opt/bro/share/broctl/scripts
	sendmail = /usr/sbin/sendmail
	sitepluginpath =
	sitepolicymanager = local-manager.bro
	sitepolicypath = /opt/bro/share/bro/site
	sitepolicystandalone = local.bro
	sitepolicyworker = local-worker.bro
	spooldir = /opt/bro/spool
	standalone = True
	statefile = /opt/bro/spool/state.db
	staticdir = /opt/bro/share/broctl
	statsdir = /opt/bro/logs/stats
	statslog = /opt/bro/spool/stats.log
	statslogenable = True
	statslogexpireinterval = 0
	statuscmdshowall = True
	stoptimeout = 60
	test.enabled = False
	test.foo = 1
	time = /usr/bin/time
	timefmt = %d %b %H:%M:%S
	timemachinehost =
	timemachineport = 47757/tcp
	tmpdir = /opt/bro/spool/tmp
	tmpexecdir = /opt/bro/spool/tmp
	tracesummary = /opt/bro/bin/trace-summary
	version = 1.4-28
	zoneid =
	-----------------------------------------------------------------------------------------------

	### NIC BUFFER CONFIG ###
	-----------------------------------------------------------------------------------------------
	echo 'net.core.rmem.max=25165824' >> /etc/sysctl.conf
	echo 'net.core.rmem.default=25165824' >> /etc/sysctl.conf
	echo 'net.core.netdev_max_backlog=1000000' >> /etc/sysctl.conf
	echo 'net.ipv4.tcp_rmem= 20480 174760 25165824' >> /etc/sysctl.conf
	sudo sysctl -p