Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
HBase Shell Perf. Test
require 'benchmark'
require 'jruby/profiler'
include Java
import java.nio.ByteBuffer
import org.apache.hadoop.hbase.CellUtil
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.HColumnDescriptor
import org.apache.hadoop.hbase.HConstants
import org.apache.hadoop.hbase.HTableDescriptor
import org.apache.hadoop.hbase.TableName
import org.apache.hadoop.hbase.client.Delete
import org.apache.hadoop.hbase.client.Get
import org.apache.hadoop.hbase.client.HBaseAdmin
import org.apache.hadoop.hbase.client.HTable
import org.apache.hadoop.hbase.client.Scan
import org.apache.hadoop.hbase.util.Bytes
# Class to hold HBase test objects and various toBytes() representations
class HBaseTestObjs
KEY = '00key0000000768'
KEY2 = '01key0000000001'
CF = 'D'
CQ = 'value'
# Create HBase test objects returning a table
def initialize()
# Initialize our handles
@keyAsBytes = Bytes.toBytes(KEY)
@key2AsBytes = Bytes.toBytes(KEY2)
@startAsBytes = Bytes.toBytes(0)
@cfAsBytes = Bytes.toBytes(CF)
@cqAsBytes = Bytes.toBytes(CQ)
@conf = HBaseConfiguration.new()
end
# Get a trivial table object
# Arguments:
# tableName - A string based table name
def table(tableName, &block)
tableNameObj = TableName.valueOf(tableName)
table = HTable.new(@conf, tableNameObj)
yield(table)
ensure
table.close
end
# Get a trivial scanner object
def scanner(tableName)
scanner = Scan.new(@keyAsBytes, @key2AsBytes)
scanner.setCacheBlocks(false)
scanner.setScanMetricsEnabled(true)
end
end
########################
# Do a simple HBase Get()
# Arguments:
# table - HTable object
# key - Bytes() representation of key
def simple_get(table, key)
# Simple Get example
res = table.get(Get.new(key))
Bytes.toString(res.value()) unless res.isEmpty()
end
## Byte size of our cells
#sizeOfResult = 30
#bb = ByteBuffer.allocate(sizeOfResult)
#
## Scanner Perf Test
#vals = []
#vals2 = []
#vals3 = []
#iterations = 10
#
#cellUtilsScan = Proc.new do
# scanner = Scan.new(keyAsBytes, key2AsBytes)
# scanner.setCacheBlocks(false)
# scanner.setScanMetricsEnabled(true)
# scanResult = table.getScanner(scanner)
# res = scanResult.next()
# while res
# cell = res.getColumnCells(cfAsBytes, cqAsBytes).first
# vals.push(Bytes.toString(CellUtil.cloneValue(cell)))
# res = scanResult.next()
# end
# metrics = scanner.getScanMetrics()
# puts "Sum Millis Nexts: #{metrics.sumOfMillisSecBetweenNexts()}"
#end
#
#bbScan = Proc.new do
# bb.position(0)
# scanner = Scan.new(keyAsBytes, key2AsBytes)
# scanner.setCacheBlocks(false)
# scanner.setScanMetricsEnabled(true)
# scanResult = table.getScanner(scanner)
# res = scanResult.next()
# while res
# bb.position(0)
# res.loadValue(cfAsBytes, cqAsBytes, bb)
# vals2.push(Bytes.toString(bb.array()[0, bb.position()]))
# res = scanResult.next()
# end
# metrics = scanner.getScanMetrics()
# puts "Sum Millis Nexts: #{metrics.sumOfMillisSecBetweenNexts()}"
##end
#
#valueScan = Proc.new do
# scanner = Scan.new(keyAsBytes, key2AsBytes)
# scanner.setCacheBlocks(false)
# scanner.setScanMetricsEnabled(true)
# scanResult = table.getScanner(scanner)
# res = scanResult.next()
# while res
# sleep(5)
# vals3.push(Bytes.toString(res.value()))
# res = scanResult.next()
# while res
# sleep(5)
# vals3.push(Bytes.toString(res.value()))
# res = scanResult.next()
# end
# metrics = scanner.getScanMetrics()
# puts "Sum Millis Nexts: #{metrics.sumOfMillisSecBetweenNexts()}"
#end
def run_test()
run_data = [cellUtilsScan, bbScan, valueScan].map do |scanProc|
{ profile => JRuby::Profiler.profile do
scanProc.call
end,
times => Benchmark.measure do
(1..iterations).each do
scanProc.call
end
end
}
end
vals.sort.eql?(vals2.sort)
vals.sort.eql?(vals3.sort)
puts "CellUtils: #{run_data[0]['times']}"
puts "BB: #{run_data[1]['times']}"
puts "Value: #{run_data[2]['times']}"
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment