cuda_tests.rb

#!/usr/bin/env ruby

require 'rubygems'
require 'logger'
require 'yaml'
require 'fileutils'
require 'csv'
require 'gruff'

module CUDA
  LOGGER = Logger.new(STDERR)

  class Profiler
    NVPROF_BIN='/Developer/NVIDIA/CUDA-6.0/bin/nvprof'

    DEFAULT_EVENTS = [
      :inst_executed,
      :sm_cta_launched,
      :uncached_global_load_transaction,
      :global_store_transaction,
      :gld_inst_32bit,
      :gld_inst_64bit,
      :gld_inst_128bit,
      :gst_inst_32bit,
      :gst_inst_64bit,
      :gst_inst_128bit,
      :threads_launched
    ]

    DEFAULT_METRICS = [
      :ipc,
      :achieved_occupancy,
      :gld_transactions,
      :gst_transactions,
      :inst_executed,
      :flops_sp
    ]

    class NotProfiled < Exception ; end
    class UnknownParamter < Exception ; end

    def initialize(binary, args=[], events=DEFAULT_EVENTS, metrics=DEFAULT_METRICS)
      @binary = binary
      @args = args
      @events = events
      @metrics = metrics
      @results = nil
    end

    def profile!(with_time=true)
      LOGGER.info("Profiling: #{@binary} #{@args.join(' ')}")

      @results = from_csv(run_profiler(nvprof_statistics))

      @results.merge!(from_csv(run_profiler(nvprof_time))) if with_time
    end

    def avg(parameter)
      value parameter, :avg
    end

    def min(parameter)
      value parameter, :min
    end

    def max(parameter)
      value parameter, :max
    end

    def results
      @results.clone
    end

    private

    def run_profiler(args)
      pipe_out, pipe_in = IO.pipe
      pid = spawn(NVPROF_BIN, *args, [:out, :err] => pipe_in)
      pipe_in.close

      Process.wait(pid)

      lines = pipe_out.readlines
      outputs = []
      append = false

      lines.each do |line|
        if line.start_with? '=='
          append = line =~ /result:/
          outputs << ""
        else
          outputs[-1] += line if append
        end
      end

      return outputs
    end

    def value(parameter, type)
      raise NotProfiled unless @results
      raise UnknownParamter unless @events.include? parameter or @metrics.include? parameter or parameter.equal? :execution_time

      @results[parameter][type]
    end

    def nvprof_statistics
      metrics = @metrics.join ','
      events = @events.join ','

      [ '-u', 's',
        '--csv',
        '--metrics', metrics,
        '--events', events,
        @binary
      ] + @args
    end

    def nvprof_time
      [ '-u', 's',
        '--csv',
        @binary
      ] + @args
    end

    def from_csv(data)
      dataset = {}

      data.each do |text|
        header = true
        indexes = { min: 0, max: 0, avg: 0, name: 0 }

        begin
          CSV.parse(text) do |row|
            next if row.empty?

            if header
              header = false

              row.each_with_index do |title, index|
                indexes.keys.each do |key|
                  indexes[key] = index if title.downcase.include? key.to_s
                end
              end

              next
            end

            begin
              name = row[indexes[:name]]

              next if name.include? ']'
              name = :execution_time if name.include? ')'

              dataset[name.to_sym] = {}
              dataset[name.to_sym][:min] = row[indexes[:min]].to_f
              dataset[name.to_sym][:max] = row[indexes[:max]].to_f
              dataset[name.to_sym][:avg] = row[indexes[:avg]].to_f
            rescue
              next
            end
          end
        rescue
          LOGGER.error 'Cannot parse CSV'
          next
        end
      end

      return dataset
    end
  end

  class Instance
    BINARY_PREFIX='./matrixMul_bench'

    attr_accessor :profiler

    def initialize(algorithm, block_size, width)
      @profiler = Profiler.new("#{BINARY_PREFIX}_#{block_size}", [
        "-width=#{width}",
        "-alg=#{algorithm}"
      ])
    end

    def run!
      @profiler.profile!

      @profiler.results
    end
  end

  class Set
    def initialize(cases)
      @cases = cases
    end

    def run!
      results = {}

      @cases.each do |specs|
        algorithm, block_sizes, widths = specs
        results[algorithm] = {}

        block_sizes.each do |block_size|
          results[algorithm][block_size] = {}

          widths.each do |width|
            results[algorithm][block_size][width] = Instance.new(algorithm, block_size, width).run!
          end
        end
      end

      return results
    end
  end

  class Table
    attr_reader :algorithm
    attr_reader :name
    attr_reader :dataset

    def initialize(name, algorithm, dataset, processor)
      LOGGER.info "Generating #{name} table for #{algorithm} algorithm"

      @name = name
      @algorithm = algorithm
      @dataset = {}

      dataset.each do |block_size, with_widths|
        with_widths.each do |width, parameters|
          @dataset[width] ||= {}

          @dataset[width][block_size] = processor.call(parameters, width, block_size)

          if @dataset[width][block_size] - @dataset[width][block_size].to_i < 0.001
            @dataset[width][block_size] = @dataset[width][block_size].to_i
          else
            @dataset[width][block_size] = @dataset[width][block_size].to_f.round(2)
          end
        end
      end
    end

    def save!
      dir = "results/#{@algorithm}"
      FileUtils.mkdir_p dir

      CSV.open("#{dir}/#{@name}.csv", "w") do |csv|
        blocks = @dataset.values.first.keys
        widths = @dataset.keys

        csv.add_row [''] + widths

        blocks.each do |block|
          csv.add_row [block] + widths.map{|w| @dataset[w][block]}
        end

      end
    end

    def self.create!(name, algorithm, dataset, with_chart = true, &processor)
      table = Table.new(name, algorithm, dataset, processor)
      table.save!

      Chart.generate! table, with_chart if with_chart

      return table
    end
  end

  class Chart
    def initialize(table)
      @name = table.name
      @algorithm = table.algorithm
      @dataset = table.dataset
    end

    def save!(max_value_given = nil)
      dir = "results/#{@algorithm}"
      FileUtils.mkdir_p dir

      chart = Gruff::Line.new(1280)
      chart.sort = false
      chart.minimum_value = 0
      chart.x_axis_label = "Matrix width"
      chart.y_axis_label = @name
      chart.theme = Gruff::Themes::PASTEL
      chart.line_width = 1.5


      blocks = @dataset.values.first.keys
      widths = @dataset.keys

      chart.labels = Hash[widths.each_with_index.map{|w, i| [i, w.to_s]}]

      max_value = 0
      blocks.each do |block_size|
        values = widths.map{|w| @dataset[w][block_size]}

        max_value = ([max_value]+values).max
      end

      chart.maximum_value = max_value*1.1
      chart.maximum_value = max_value_given if max_value_given.is_a? Float

      chart.y_axis_increment = chart.maximum_value / 10.0

      blocks.each do |block_size|
        chart.data block_size, widths.map{|w| @dataset[w][block_size]}
      end

      chart.write "#{dir}/#{@name}.png"
    end

    def self.generate!(table, max_value = nil)
      chart = Chart.new(table)
      chart.save! max_value

      return chart
    end
  end

  class SummaryChart
    def initialize(tables)
      @name = tables.first.name
      @tables = tables
      @instance = tables.map{|t| t.dataset.keys.max}.min
      @blocks = [tables.first.dataset.values.first.keys[0], tables.first.dataset.values.first.keys[-1]]
    end

    def save!
      dir = "results/"
      FileUtils.mkdir_p dir

      chart = Gruff::Bar.new(1280)
      chart.sort = false
      chart.y_axis_label = @name
      chart.x_axis_label = "Matrix Multiplication Algorithm (instance=#{@instance})"
      chart.theme = Gruff::Themes::PASTEL

      chart.labels = Hash[@tables.each_with_index.map{|t,i| [i, t.algorithm.to_s]}]

      @blocks.each do |block|
        chart.data block.to_s, @tables.map{|t| t.dataset[@instance][block]}
      end

      chart.write "#{dir}/#{@name}_a#{@instance}_#{@tables.map(&:algorithm).join('_')}.png"
    end

    def self.generate!(tables)
      chart = SummaryChart.new(tables)
      chart.save!

      return chart
    end
  end

  def self.run!
    results = nil


    begin
      results = YAML.load_file('results.yml')

      LOGGER.info 'Loaded tests results'
    rescue
      LOGGER.info 'Conducting new tests'

      widths = Array.new(4).each_with_index.map{|x,i| 256 * (i + 1)}

      algorithms = [
        [3, [8, 16, 32], widths],
        [4, [8, 16, 32], widths],
        [1, [8, 16, 32], [64, 128, 192, 256]],
        [2, [8, 16, 32], widths],
      ]

      set = Set.new(algorithms)

      results = set.run!

      File.open('results.yml', 'w') do |file|
        file.write results.to_yaml
      end
    end

    FileUtils.rm_rf './results'

    tables = {
      execution_time: {},
      gflops: {},
      cgma: {}
    }

    results.each do |algorithm, dataset|
      tables[:execution_time][algorithm] = Table.create!("Execution Time [ms]", algorithm, dataset) do |results, width, block_size|
        results[:execution_time][:avg] * 1000
      end

      tables[:gflops][algorithm] = Table.create!("GFLOPS", algorithm, dataset) do |results, width, block_size|
        2*width**3 / results[:execution_time][:avg] / 10**9
      end

      Table.create! "MIPS", algorithm, dataset do |results, width, block_size|
        results[:inst_executed][:avg] / results[:execution_time][:avg] / 10**6
      end

      tables[:cgma][algorithm] = Table.create!("CGMA", algorithm, dataset) do |results, width, block_size|
        results[:inst_executed][:avg] / results[:uncached_global_load_transaction][:avg]
      end

      Table.create! "IPC", algorithm, dataset do |results, width, block_size|
        results[:ipc][:avg]
      end

      Table.create! "Achieved occupancy", algorithm, dataset, 1.0 do |results, width, block_size|
        results[:achieved_occupancy][:avg]
      end

      Table.create! "Executed millions instructions", algorithm, dataset do |results, width, block_size|
        results[:inst_executed][:avg] / 10**6
      end

      Table.create! "Transmissions (millions)", algorithm, dataset do |results, width, block_size|
        ( results[:gld_inst_32bit][:avg] + results[:gld_inst_64bit][:avg] + results[:gld_inst_128bit][:avg] +
          results[:gst_inst_32bit][:avg] + results[:gst_inst_64bit][:avg] + results[:gst_inst_128bit][:avg] ) / 10**6
      end

      Table.create! "Transmissions transactions (millions)", algorithm, dataset do |results, width, block_size|
        ( results[:global_store_transaction][:avg] + results[:uncached_global_load_transaction][:avg] ) / 10**6
      end
    end

    tables.keys.each do |key|
      SummaryChart.generate! [tables[key][1], tables[key][2], tables[key][3], tables[key][4]]
      SummaryChart.generate! [tables[key][2], tables[key][3], tables[key][4]]
    end
  end
end

CUDA::run! if __FILE__ == $0