Skip to content

Instantly share code, notes, and snippets.

@1tylermitchell
Last active May 9, 2018 23:03
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save 1tylermitchell/a58ccec3f55e35f9ed8a to your computer and use it in GitHub Desktop.
Spec file for checking various Hadoop/HDFS cluster settings using serverspec platform
require 'spec_helper'
require 'nokogiri' # For XML parsing of config files
describe "Processor count" do
# I hate calling out directly from Ruby like this
# but the function: "command('nproc')" was returning strings I couldn't cast
describe `nproc`.to_i do
it { should >= 12 }
end
end
describe "Ulimits" do
describe command('ulimit') do
its(:value) { should be >= 65000 }
end
end
describe "Hadoop packages exist" do
# Disabling package function method as name is too version specific
# Need a way to generalise the check for package on non Centos
#describe package('hadoop_2_2_0_0_2041') do
# it { should be_installed }
#end
describe `rpm -qa hadoop* | wc -l`.to_i do
it { should be > 0 }
end
end
describe "Hadoop service running" do
# Check that this works on non Centos environments
describe service('hadoop') do
it { should be_running }
end
end
describe "HDFS Configuration" do
# Checking XML file via bash, replaced by XML nokogiri lib further down
#describe command('grep dfs.replication /etc/hadoop/conf/hdfs-site.xml -A1 | tail -n1 | sed "s/\(^[ \t]*\|<\|>\)/\t/g" | cut -f4'.to_i) do
# it { should be > 0 }
#end
describe "dfs.replication" do
describe file('/etc/hadoop/conf/hdfs-site.xml') do
#it {should be_file}
it {should contain /dfs.replication/}
it do
doc = Nokogiri::XML(File.read("/etc/hadoop/conf/hdfs-site.xml"))
d=doc.xpath("//property[name='dfs.replication']/value").text.to_i
expect(d).to be >= 1
end
end
end
end
# Checking XML file via bash, replaced by XML nokogiri lib further down
# describe `grep dfs.client.read.shortcircuit.streams.cache.size /etc/hadoop/conf/hdfs-site.xml -A1 | tail -n1 | sed "s/\(^[ \t]*\|<\|>\)/\t/g" | cut -f4`.to_i do
# it { should be >= 4096 }
#end
describe "streams.cache.size" do
describe file('/etc/hadoop/conf/hdfs-site.xml') do
#it {should be_file}
it {should contain /dfs.client.read.shortcircuit.streams.cache.size/}
it do
doc = Nokogiri::XML(File.read("/etc/hadoop/conf/hdfs-site.xml"))
d=doc.xpath("//property[name='dfs.client.read.shortcircuit.streams.cache.size']/value").text.to_i
expect(d).to be >= 4096
end
end
end
describe "max.transfer.threads" do
describe file('/etc/hadoop/conf/hdfs-site.xml') do
#it {should be_file}
it {should contain /dfs.datanode.max.transfer.threads/}
it do
doc = Nokogiri::XML(File.read("/etc/hadoop/conf/hdfs-site.xml"))
d=doc.xpath("//property[name='dfs.datanode.max.transfer.threads']/value").text.to_i
expect(d).to be >= 4096
end
end
end
end
end
describe "Disk count" do
describe `lsblk -a | grep disk | wc -l`.to_i do
it { should be >= 12 }
end
end
describe "HDFS Ports" do
describe "Data UI Port" do
describe port(50075) do
it { should be_listening }
end
end
describe "Data Transfer Port" do
describe port(50010) do
it { should be_listening }
end
end
describe "Name node UI Port" do
describe port(50070) do
it { should be_listening }
end
end
end
describe "Node Memory Availability 128/256GB" do
describe host_inventory['memory']['total'].delete('kB').to_i do
it { should >= 128000000 }
end
describe host_inventory['memory']['total'].delete('kB').to_i do
it { should >= 256000000 }
end
end
describe "Network Interfaces" do
describe interface('eth0') do
its(:speed) { should be >= 10000 }
end
describe interface('eth0') do
its(:speed) { should be >= 1000 }
end
describe interface('eth1') do
its(:speed) { should be >= 10000 }
end
describe interface('eth1') do
its(:speed) { should be >= 1000 }
end
end
describe "THP Disabled" do
describe command('cat /sys/kernel/mm/redhat_transparent_hugepage/enabled') do
its(:stdout) { should contain('[never]') }
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment