Kernel entries for Hadoop

kernel-entries-for-hadoop.md

Kernel entries for Hadoop - Tuning

Should be added to either /etc/sysctl.conf or /etc/sysct.conf.d/99-hadoop.conf

File contents:

# Sysctl for hadoop nodes

net.core.netdev_max_backlog = 4000
net.core.somaxconn = 4000

# network stack tuning for high throughput low latency links
net.ipv4.ip_forward = 0
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_sack = 0
net.ipv4.tcp_dsack = 0
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 5
net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp_fin_timeout = 30
net.ipv4.tcp_rmem = 32768 436600 4193404
net.ipv4.tcp_wmem = 32768 436600 4193404
net.ipv4.tcp_retries2 = 10 
net.ipv4.tcp_synack_retries = 3

# ARP table sizes.  Important for >500 server L2 networks. 
net.ipv4.neigh.default.gc_interval = 3600
net.ipv4.neigh.default.gc_stale_time = 3600
net.ipv4.neigh.default.gc_thresh3 = 5120
net.ipv4.neigh.default.gc_thresh2 = 4096
net.ipv4.neigh.default.gc_thresh1 = 2048

# disable ipv6 – optional but useful to avoid troubleshooting issues.
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1

kernel.sysrq = 0
kernel.core_uses_pid = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296

# avoid swapping
vm.swappiness = 0

# Tune memory cache
vm.dirty_ratio=50
vm.dirty_background_ratio=20

After adding content to file, you need to update kernel table:

sysctl -p

hi tanx for sharing, what is usecase of these two line in hadoop field
kernel.sysrq = 0
kernel.core_uses_pid = 1
?