HBase Shell Perf. Test

hbase_shell_perf_test.rb

require 'benchmark'
require 'jruby/profiler'
include Java
import java.nio.ByteBuffer
import org.apache.hadoop.hbase.CellUtil
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.HColumnDescriptor
import org.apache.hadoop.hbase.HConstants
import org.apache.hadoop.hbase.HTableDescriptor
import org.apache.hadoop.hbase.TableName
import org.apache.hadoop.hbase.client.Delete
import org.apache.hadoop.hbase.client.Get
import org.apache.hadoop.hbase.client.HBaseAdmin
import org.apache.hadoop.hbase.client.HTable
import org.apache.hadoop.hbase.client.Scan
import org.apache.hadoop.hbase.util.Bytes

#  Class to hold HBase test objects and various toBytes() representations
class HBaseTestObjs
  KEY = '00key0000000768'
  KEY2 = '01key0000000001'
  CF = 'D'
  CQ = 'value'

  # Create HBase test objects returning a table
  def initialize()
    # Initialize our handles
    @keyAsBytes = Bytes.toBytes(KEY)
    @key2AsBytes = Bytes.toBytes(KEY2)
    @startAsBytes = Bytes.toBytes(0)
    @cfAsBytes = Bytes.toBytes(CF)
    @cqAsBytes = Bytes.toBytes(CQ)
    @conf = HBaseConfiguration.new()
  end

  # Get a trivial table object
  # Arguments:
  #   tableName - A string based table name
  def table(tableName, &block)
    tableNameObj = TableName.valueOf(tableName)
    table = HTable.new(@conf, tableNameObj)
    yield(table)
  ensure
    table.close
  end

  # Get a trivial scanner object
  def scanner(tableName)
    scanner = Scan.new(@keyAsBytes, @key2AsBytes)
    scanner.setCacheBlocks(false)
    scanner.setScanMetricsEnabled(true)
  end
end

########################
# Do a simple HBase Get()
# Arguments:
#   table - HTable object
#     key - Bytes() representation of key
def simple_get(table, key)
  # Simple Get example
  res = table.get(Get.new(key))
  Bytes.toString(res.value()) unless res.isEmpty()
end

## Byte size of our cells
#sizeOfResult = 30
#bb = ByteBuffer.allocate(sizeOfResult)
#
## Scanner Perf Test
#vals = []
#vals2 = []
#vals3 = []
#iterations = 10
#
#cellUtilsScan = Proc.new do
#  scanner = Scan.new(keyAsBytes, key2AsBytes)
#  scanner.setCacheBlocks(false)
#  scanner.setScanMetricsEnabled(true)
#  scanResult = table.getScanner(scanner)
#  res = scanResult.next()
#  while res
#    cell = res.getColumnCells(cfAsBytes, cqAsBytes).first
#    vals.push(Bytes.toString(CellUtil.cloneValue(cell)))
#    res = scanResult.next()
#  end
#  metrics = scanner.getScanMetrics()
#  puts "Sum Millis Nexts: #{metrics.sumOfMillisSecBetweenNexts()}"
#end
#
#bbScan = Proc.new do
#  bb.position(0)
#  scanner = Scan.new(keyAsBytes, key2AsBytes)
#  scanner.setCacheBlocks(false)
#  scanner.setScanMetricsEnabled(true)
#  scanResult = table.getScanner(scanner)
#  res = scanResult.next()
#  while res
#    bb.position(0)
#    res.loadValue(cfAsBytes, cqAsBytes, bb)
#    vals2.push(Bytes.toString(bb.array()[0, bb.position()]))
#    res = scanResult.next()
#  end
#  metrics = scanner.getScanMetrics()
#  puts "Sum Millis Nexts: #{metrics.sumOfMillisSecBetweenNexts()}"
##end
#
#valueScan = Proc.new do
#  scanner = Scan.new(keyAsBytes, key2AsBytes)
#  scanner.setCacheBlocks(false)
#  scanner.setScanMetricsEnabled(true)
#  scanResult = table.getScanner(scanner)
#  res = scanResult.next()
#  while res
#    sleep(5)
#    vals3.push(Bytes.toString(res.value()))
#    res = scanResult.next()
#  while res
#    sleep(5)
#    vals3.push(Bytes.toString(res.value()))
#    res = scanResult.next()
#  end
#  metrics = scanner.getScanMetrics()
#  puts "Sum Millis Nexts: #{metrics.sumOfMillisSecBetweenNexts()}"
#end


def run_test()
  run_data = [cellUtilsScan, bbScan, valueScan].map do |scanProc|
    { profile => JRuby::Profiler.profile do
        scanProc.call
      end,
      times => Benchmark.measure do
        (1..iterations).each do
          scanProc.call
        end
      end
    }
  end

  vals.sort.eql?(vals2.sort)
  vals.sort.eql?(vals3.sort)
  puts "CellUtils: #{run_data[0]['times']}"
  puts "BB: #{run_data[1]['times']}"
  puts "Value: #{run_data[2]['times']}"
end