NickLaMuro/README.md Secret

## README.md

      
    Raw
  

              README.md
            
          
    Collection of scripts to report on memory usage in the application after boot
and for individual dependencies.  So of these are very [WIP] at the moment,
and don't fully do what we want it to... yet.
Usage

Take the scripts from this repo and plug them into the bin/ directory of
manageiq.  You can run them from there doing bin/[script_name].
Scripts

benchmark

Benchmarks the total memory usage of a single gem dependency, or the entire
rails process.
If no arguments are passed, the script will benchmark loading the rails
application as if it was loaded, and print out the memory total memory used
to do so.  If 1 or more arguments are passed in, it will benchmark loading
those as gems/require paths individually, and then print out the memory used to
do that.
benchmark_branch

Checks out the specific tag, updates the gems, resets the database, and runs
bin/benchmark 5 times and saves it to a file.  Does this in separate
processes to make sure the process is reloaded fully each time (that said,
because of hardware caching, this number tends to go down over time).
benchmark_gems

Checks out the specific tag, updates the gems, parses the gemfile for a list of
direct dependencies, and runs bin/benchmark [GEM] for each gem that was
parsed, and saves the results to a file.
gemfile_top_level_dependency_parser

Parses a Gemfile to determine a list of gems that are direct dependencies and
are not part of the :development/:test groups.  Probably could have been
done with some of the bundler code directly, but I couldn't be bothered digging
through the source when stubbing some methods was just as easy.  Worth finding
a different way of doing this I suppose, but did the trick for now.

  
## benchmark
#!/usr/bin/env ruby

# This is a modified version of the derailed_benchmarks lib, originally
# authored by Richard Schneeman (@schneems):
#
#   https://github.com/schneems/derailed_benchmarks
#
# No current license exists.

# Usage: benchmark [REQUIRE_STRING...]
#
# If no arguments are passed, the script will benchmark loading the rails
# application as if it was loaded, and print out the memory total memory used
# to do so.
#
# If 1 or more arguments are passed in, it will benchmark loading those as
# gems/require paths individually, and then print out the memory used to do
# that.

require 'rubygems'

ENV['CUT_OFF']                = "0.0"
ENV['LOG_LEVEL']              = "FATAL"
ENV["RAILS_ENV"]              = "production"
ENV['RACK_ENV']               = ENV["RAILS_ENV"]
ENV["DISABLE_SPRING"]         = "true"
ENV['RAILS_USE_MEMORY_STORE'] = "1"
ENV['ERB_IN_CONFIG']          = "1"

ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)

require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
require 'rails/all' if ARGV.length > 0 && !ARGV.include?('rails')

require 'yaml'
require 'bigdecimal'
require 'sys/proctable'
require 'English'

module MiqBenchmark
  class RequireTree
    attr_reader   :name
    attr_accessor :cost
    attr_accessor :parent

    def self.required_by
      @required_by ||= {}
    end

    def initialize(name)
      @name     = name
      @children = {}
    end

    def <<(tree)
      @children[tree.name.to_s] = tree
      tree.parent = self
      (self.class.required_by[tree.name.to_s] ||= []) << name
    end

    def [](name)
      @children[name.to_s]
    end

    # Returns array of child nodes
    def children
      @children.values
    end

    def cost
      @cost || 0
    end

    def short_name
      @short_name ||= name.gsub(/^#{Dir.home}.*\/bundler\/gems\//, '')
                          .gsub(/^([^\/]+)-[0-9a-f]+\//, '\1/')
                          .gsub(/^#{defined?(Rails) ? Rails.root : Dir.pwd}\//, '')
    end

    # Returns sorted array of child nodes from Largest to Smallest
    def sorted_children
      children.sort { |c1, c2| c2.cost <=> c1.cost }
    end

    def to_string
      str = "#{short_name}: #{cost.round(4)} MiB"
      if parent && self.class.required_by[name.to_s]
        names = self.class.required_by[name.to_s].uniq - [parent.name.to_s]
        if names.any?
          str << " (Also required by: #{names.first(2).join(", ")}"
          str << ", and #{names.count - 2} others" if names.count > 3
          str << ")"
        end
      end
      str
    end

    # Recursively prints all child nodes
    def print_sorted_children(level = 0, out = STDOUT)
      return if cost < ENV['CUT_OFF'].to_f
      out.puts "  " * level + to_string
      level += 1
      sorted_children.each do |child|
        child.print_sorted_children(level, out)
      end
    end

    def print_summary(out = STDOUT)
      longest_name = sorted_children.map { |c| c.short_name.length }.max
      longest_cost = sorted_children.map { |c| c.cost.round(4).to_s.length }.max

      out.puts "\n\n\n"
      out.puts "SUMMARY ( TOTAL COST: #{cost.round(4)} MiB )"
      out.puts "-" * (longest_name + longest_cost + 7)

      sorted_children.each do |child|
        next if child.cost < ENV['CUT_OFF'].to_f
        out.puts [
          child.short_name.ljust(longest_name),
          "#{"%.4f" % child.cost} MiB".rjust(longest_cost + 4)
        ].join(' | ')
      end
    end
  end
end

module Kernel
  alias original_require require

  MB_BYTES = ::BigDecimal.new(1_048_576)
  if ENV["CI"] && ENV["TRAVIS"]
    # Travis doesn't seem to like Sys::ProcTable very much... this is the work
    # around for now until we can get that worked out.
    CONVERSION       = {"kb" => 1024, "mb" => 1_048_576, "gb" => 1_073_741_824}.freeze
    PROC_STATUS_FILE = Pathname.new("/proc/#{$PID}/status").freeze
    VMRSS_GREP_EXP   = /^VmRSS/
    MEMORY_MB_PROC = proc do
      begin
        rss_line = PROC_STATUS_FILE.each_line.grep(VMRSS_GREP_EXP).first
        return unless rss_line
        return unless (_name, value, unit = rss_line.split(nil)).length == 3
        (CONVERSION[unit.downcase!] * ::BigDecimal.new(value)) / MB_BYTES
      rescue Errno::EACCES, Errno::ENOENT
        0
      end
    end
  else
    MEMORY_MB_PROC = proc { (Sys::ProcTable.ps($PID).rss / MB_BYTES).to_f }
  end

  def require(file)
    Kernel.require(file)
  end

  # This breaks things, not sure how to fix
  # def require_relative(file)
  #   Kernel.require_relative(file)
  # end
  class << self
    attr_writer :require_stack
    alias original_require require
    # alias :original_require_relative :require_relative

    def require_stack
      @require_stack ||= []
    end
  end

  def self.measure_memory_impact(file, &block)
    node   = MiqBenchmark::RequireTree.new(file)

    parent = require_stack.last
    parent << node
    require_stack.push(node)
    begin
      before = ::Kernel::MEMORY_MB_PROC.call
      block.call file
    ensure
      require_stack.pop # node
      after = ::Kernel::MEMORY_MB_PROC.call
    end
    node.cost = after - before
  end
end

# Top level node that will store all require information for the entire app
TOP_REQUIRE = MiqBenchmark::RequireTree.new("TOP")
::Kernel.require_stack.push(TOP_REQUIRE)

Kernel.define_singleton_method(:require) do |file|
  measure_memory_impact(file) { |f| original_require(f) }
end

# Don't forget to assign a cost to the top level
cost_before_requiring_anything = ::Kernel::MEMORY_MB_PROC.call
TOP_REQUIRE.cost = cost_before_requiring_anything
def TOP_REQUIRE.set_top_require_cost
  self.cost = ::Kernel::MEMORY_MB_PROC.call - cost
end

if ARGV.length > 0
  ARGV.each do |arg|
    libdir_files = Dir.glob(Gem::Specification.find_by_name(arg).lib_dirs_glob + "/*.rb")
    # Check one more level down
    libdir_files = Dir.glob(Gem::Specification.find_by_name(arg).lib_dirs_glob + "/*/*.rb") if libdir_files.empty?

    if libdir_files.empty?
      require arg
    else
      libdir_files.each {|f| require f }
    end
  end

  TOP_REQUIRE.set_top_require_cost
  puts "#{ARGV.join(' ')} => #{TOP_REQUIRE.cost.round(4)} MiB"
else
  require ::File.expand_path('../../config/application', __FILE__)

  Vmdb::Application.configure do
    config.instance_variable_set(:@eager_load, true)
  end

  # Prevent overriding the @eager_load variable in the `config/environments/production.rb`
  class Rails::Application::Configuration
    def eager_load=(value)
    end

    def eager_load_paths=(value)
    end
  end

  Vmdb::Application.initialize!

  TOP_REQUIRE.set_top_require_cost
  puts "#{TOP_REQUIRE.cost.round(4)} MiB"
end

## benchmark_branch
#!/bin/bash

# Usage:  benchmark_branch <git_tag>
#
# Checks out the specific tag, updates the gems, resets the database, and runs
# `bin/benchmark` 5 times and saves it to a file.  Does this in separate
# processes to make sure the process is reloaded fully each time (that said,
# because of hardware caching, this number tends to go down over time).


# You can determine which tag to benchmark for a given release using the
# following:
#
#     git tag | grep "^5.6" | grep -v -e alpha -e beta -e rc
#

git checkout $1

# Re-bundle
bin/bundle update > /dev/null

# Setup the DB
DISABLE_DATABASE_ENVIRONMENT_CHECK=1 \
RAILS_USE_MEMORY_STORE=1 \
RAILS_ENV=production \
ERB_IN_CONFIG=1 \
bin/rake db:environment:set db:drop db:create db:migrate > /dev/null

# Run the benchmarks five times
seq 1 5 | xargs -I {} /bin/bash -c 'bin/benchmark >> "tmp/miq_benchmark/$(git describe --tags)"'

## benchmark_gems
#!/bin/bash

# Usage:  benchmark_gems <git_tag>
#
# Checks out the specific tag, updates the gems, parses the gemfile for a list
# of direct dependencies, and runs `bin/benchmark [GEM]` for each gem that was
# parsed, and saves the results to a file.


# You can determine which tag to benchmark for a given release using the
# following:
#
#     git tag | grep "^5.6" | grep -v -e alpha -e beta -e rc
#

git checkout $1

# Re-bundle
echo "Running 'bin/bundle update'..."
bin/bundle update > /dev/null

# Run the benchmarks five times
echo "Benchmarking..."
OUTPUT_FILE="tmp/miq_benchmark/gems/$(git describe --tags)"
bin/gemfile_top_level_dependency_parser | while read gem; do
  bin/benchmark $gem >> $OUTPUT_FILE
  # seq 1 5 | xargs -I {} /bin/bash -c \
  #   "(printf $gem && bin/benchmark $gem )>> $OUTPUT_FILE"
done

## gemfile_top_level_dependency_parser
#!/usr/bin/env ruby

class GemfileTopLevelDepParser
  attr_accessor :extra_gemfiles_to_eval

  DASH_TO_SLASH = {
    "fog-google"          => "fog/google",
    "fog-openstack"       => "fog/openstack",
    "fog-vcloud-director" => "fog/vcloud_director",
    "google-api-client"   => "google/api_client",
    "hawkular-client"     => "hawkular/hawkular_client",
    "net-ping"            => "net/ping",
    "net-scp"             => "net/scp",
    "net-sftp"            => "net/sftp",
    "net-ssh"             => "net/ssh",
    "websocket-driver"    => "websocket/driver",
    "zip-zip"             => "zip/zip"
  }

  class << self
    attr_accessor :top_level_dependencies

    def parse gemfile
      new.tap {|i| i.instance_eval { eval File.read(gemfile) } }
         .tap {|i|
           i.extra_gemfiles_to_eval.each {|gf|
             i.instance_eval { eval File.read(gf) }
           }
         }
      # puts convert_top_level_to_require_strings
      puts top_level_dependencies
    end

    def convert_top_level_to_require_strings
      top_level_dependencies.map do |entry|
        DASH_TO_SLASH[entry] || entry
      end
    end
  end

  def initialize
    @extra_gemfiles_to_eval             = []
    self.class.top_level_dependencies ||= []
  end

  def gem *args
    self.class.top_level_dependencies << args.first
  end

  # Only parse other gemfiles if they are from gems pending, otherwise ignore
  def eval_gemfile filename
    if filename.include? "gems/pending/Gemfile"
      gems_pending_gemfile = File.expand_path "../../gems/pending/Gemfile", __FILE__
      extra_gemfiles_to_eval << gems_pending_gemfile if File.exist? gems_pending_gemfile
    end
  end

  def group *types
    unless types.include? :development or types.include? :test
      yield if block_given?
    end
  end

  def dependencies
    []
  end

  def source *args; end;
end


GemfileTopLevelDepParser.parse File.expand_path "../../Gemfile", __FILE__
	#!/usr/bin/env ruby

	# This is a modified version of the derailed_benchmarks lib, originally
	# authored by Richard Schneeman (@schneems):
	#
	# https://github.com/schneems/derailed_benchmarks
	#
	# No current license exists.

	# Usage: benchmark [REQUIRE_STRING...]
	#
	# If no arguments are passed, the script will benchmark loading the rails
	# application as if it was loaded, and print out the memory total memory used
	# to do so.
	#
	# If 1 or more arguments are passed in, it will benchmark loading those as
	# gems/require paths individually, and then print out the memory used to do
	# that.

	require 'rubygems'

	ENV['CUT_OFF'] = "0.0"
	ENV['LOG_LEVEL'] = "FATAL"
	ENV["RAILS_ENV"] = "production"
	ENV['RACK_ENV'] = ENV["RAILS_ENV"]
	ENV["DISABLE_SPRING"] = "true"
	ENV['RAILS_USE_MEMORY_STORE'] = "1"
	ENV['ERB_IN_CONFIG'] = "1"

	ENV['BUNDLE_GEMFILE'] \|\|= File.expand_path('../../Gemfile', __FILE__)

	require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
	require 'rails/all' if ARGV.length > 0 && !ARGV.include?('rails')

	require 'yaml'
	require 'bigdecimal'
	require 'sys/proctable'
	require 'English'

	module MiqBenchmark
	class RequireTree
	attr_reader :name
	attr_accessor :cost
	attr_accessor :parent

	def self.required_by
	@required_by \|\|= {}
	end

	def initialize(name)
	@name = name
	@children = {}
	end

	def <<(tree)
	@children[tree.name.to_s] = tree
	tree.parent = self
	(self.class.required_by[tree.name.to_s] \|\|= []) << name
	end

	def [](name)
	@children[name.to_s]
	end

	# Returns array of child nodes
	def children
	@children.values
	end

	def cost
	@cost \|\| 0
	end

	def short_name
	@short_name \|\|= name.gsub(/^#{Dir.home}.*\/bundler\/gems\//, '')
	.gsub(/^([^\/]+)-[0-9a-f]+\//, '\1/')
	.gsub(/^#{defined?(Rails) ? Rails.root : Dir.pwd}\//, '')
	end

	# Returns sorted array of child nodes from Largest to Smallest
	def sorted_children
	children.sort { \|c1, c2\| c2.cost <=> c1.cost }
	end

	def to_string
	str = "#{short_name}: #{cost.round(4)} MiB"
	if parent && self.class.required_by[name.to_s]
	names = self.class.required_by[name.to_s].uniq - [parent.name.to_s]
	if names.any?
	str << " (Also required by: #{names.first(2).join(", ")}"
	str << ", and #{names.count - 2} others" if names.count > 3
	str << ")"
	end
	end
	str
	end

	# Recursively prints all child nodes
	def print_sorted_children(level = 0, out = STDOUT)
	return if cost < ENV['CUT_OFF'].to_f
	out.puts " " * level + to_string
	level += 1
	sorted_children.each do \|child\|
	child.print_sorted_children(level, out)
	end
	end

	def print_summary(out = STDOUT)
	longest_name = sorted_children.map { \|c\| c.short_name.length }.max
	longest_cost = sorted_children.map { \|c\| c.cost.round(4).to_s.length }.max

	out.puts "\n\n\n"
	out.puts "SUMMARY ( TOTAL COST: #{cost.round(4)} MiB )"
	out.puts "-" * (longest_name + longest_cost + 7)

	sorted_children.each do \|child\|
	next if child.cost < ENV['CUT_OFF'].to_f
	out.puts [
	child.short_name.ljust(longest_name),
	"#{"%.4f" % child.cost} MiB".rjust(longest_cost + 4)
	].join(' \| ')
	end
	end
	end
	end

	module Kernel
	alias original_require require

	MB_BYTES = ::BigDecimal.new(1_048_576)
	if ENV["CI"] && ENV["TRAVIS"]
	# Travis doesn't seem to like Sys::ProcTable very much... this is the work
	# around for now until we can get that worked out.
	CONVERSION = {"kb" => 1024, "mb" => 1_048_576, "gb" => 1_073_741_824}.freeze
	PROC_STATUS_FILE = Pathname.new("/proc/#{$PID}/status").freeze
	VMRSS_GREP_EXP = /^VmRSS/
	MEMORY_MB_PROC = proc do
	begin
	rss_line = PROC_STATUS_FILE.each_line.grep(VMRSS_GREP_EXP).first
	return unless rss_line
	return unless (_name, value, unit = rss_line.split(nil)).length == 3
	(CONVERSION[unit.downcase!] * ::BigDecimal.new(value)) / MB_BYTES
	rescue Errno::EACCES, Errno::ENOENT
	0
	end
	end
	else
	MEMORY_MB_PROC = proc { (Sys::ProcTable.ps($PID).rss / MB_BYTES).to_f }
	end

	def require(file)
	Kernel.require(file)
	end

	# This breaks things, not sure how to fix
	# def require_relative(file)
	# Kernel.require_relative(file)
	# end
	class << self
	attr_writer :require_stack
	alias original_require require
	# alias :original_require_relative :require_relative

	def require_stack
	@require_stack \|\|= []
	end
	end

	def self.measure_memory_impact(file, &block)
	node = MiqBenchmark::RequireTree.new(file)

	parent = require_stack.last
	parent << node
	require_stack.push(node)
	begin
	before = ::Kernel::MEMORY_MB_PROC.call
	block.call file
	ensure
	require_stack.pop # node
	after = ::Kernel::MEMORY_MB_PROC.call
	end
	node.cost = after - before
	end
	end

	# Top level node that will store all require information for the entire app
	TOP_REQUIRE = MiqBenchmark::RequireTree.new("TOP")
	::Kernel.require_stack.push(TOP_REQUIRE)

	Kernel.define_singleton_method(:require) do \|file\|
	measure_memory_impact(file) { \|f\| original_require(f) }
	end

	# Don't forget to assign a cost to the top level
	cost_before_requiring_anything = ::Kernel::MEMORY_MB_PROC.call
	TOP_REQUIRE.cost = cost_before_requiring_anything
	def TOP_REQUIRE.set_top_require_cost
	self.cost = ::Kernel::MEMORY_MB_PROC.call - cost
	end

	if ARGV.length > 0
	ARGV.each do \|arg\|
	libdir_files = Dir.glob(Gem::Specification.find_by_name(arg).lib_dirs_glob + "/*.rb")
	# Check one more level down
	libdir_files = Dir.glob(Gem::Specification.find_by_name(arg).lib_dirs_glob + "//.rb") if libdir_files.empty?

	if libdir_files.empty?
	require arg
	else
	libdir_files.each {\|f\| require f }
	end
	end

	TOP_REQUIRE.set_top_require_cost
	puts "#{ARGV.join(' ')} => #{TOP_REQUIRE.cost.round(4)} MiB"
	else
	require ::File.expand_path('../../config/application', __FILE__)

	Vmdb::Application.configure do
	config.instance_variable_set(:@eager_load, true)
	end

	# Prevent overriding the @eager_load variable in the `config/environments/production.rb`
	class Rails::Application::Configuration
	def eager_load=(value)
	end

	def eager_load_paths=(value)
	end
	end

	Vmdb::Application.initialize!

	TOP_REQUIRE.set_top_require_cost
	puts "#{TOP_REQUIRE.cost.round(4)} MiB"
	end
	#!/bin/bash

	# Usage: benchmark_branch <git_tag>
	#
	# Checks out the specific tag, updates the gems, resets the database, and runs
	# `bin/benchmark` 5 times and saves it to a file. Does this in separate
	# processes to make sure the process is reloaded fully each time (that said,
	# because of hardware caching, this number tends to go down over time).


	# You can determine which tag to benchmark for a given release using the
	# following:
	#
	# git tag \| grep "^5.6" \| grep -v -e alpha -e beta -e rc
	#

	git checkout $1

	# Re-bundle
	bin/bundle update > /dev/null

	# Setup the DB
	DISABLE_DATABASE_ENVIRONMENT_CHECK=1 \
	RAILS_USE_MEMORY_STORE=1 \
	RAILS_ENV=production \
	ERB_IN_CONFIG=1 \
	bin/rake db:environment:set db:drop db:create db:migrate > /dev/null

	# Run the benchmarks five times
	seq 1 5 \| xargs -I {} /bin/bash -c 'bin/benchmark >> "tmp/miq_benchmark/$(git describe --tags)"'
	#!/bin/bash

	# Usage: benchmark_gems <git_tag>
	#
	# Checks out the specific tag, updates the gems, parses the gemfile for a list
	# of direct dependencies, and runs `bin/benchmark [GEM]` for each gem that was
	# parsed, and saves the results to a file.


	# You can determine which tag to benchmark for a given release using the
	# following:
	#
	# git tag \| grep "^5.6" \| grep -v -e alpha -e beta -e rc
	#

	git checkout $1

	# Re-bundle
	echo "Running 'bin/bundle update'..."
	bin/bundle update > /dev/null

	# Run the benchmarks five times
	echo "Benchmarking..."
	OUTPUT_FILE="tmp/miq_benchmark/gems/$(git describe --tags)"
	bin/gemfile_top_level_dependency_parser \| while read gem; do
	bin/benchmark $gem >> $OUTPUT_FILE
	# seq 1 5 \| xargs -I {} /bin/bash -c \
	# "(printf $gem && bin/benchmark $gem )>> $OUTPUT_FILE"
	done
	#!/usr/bin/env ruby

	class GemfileTopLevelDepParser
	attr_accessor :extra_gemfiles_to_eval

	DASH_TO_SLASH = {
	"fog-google" => "fog/google",
	"fog-openstack" => "fog/openstack",
	"fog-vcloud-director" => "fog/vcloud_director",
	"google-api-client" => "google/api_client",
	"hawkular-client" => "hawkular/hawkular_client",
	"net-ping" => "net/ping",
	"net-scp" => "net/scp",
	"net-sftp" => "net/sftp",
	"net-ssh" => "net/ssh",
	"websocket-driver" => "websocket/driver",
	"zip-zip" => "zip/zip"
	}

	class << self
	attr_accessor :top_level_dependencies

	def parse gemfile
	new.tap {\|i\| i.instance_eval { eval File.read(gemfile) } }
	.tap {\|i\|
	i.extra_gemfiles_to_eval.each {\|gf\|
	i.instance_eval { eval File.read(gf) }
	}
	}
	# puts convert_top_level_to_require_strings
	puts top_level_dependencies
	end

	def convert_top_level_to_require_strings
	top_level_dependencies.map do \|entry\|
	DASH_TO_SLASH[entry] \|\| entry
	end
	end
	end

	def initialize
	@extra_gemfiles_to_eval = []
	self.class.top_level_dependencies \|\|= []
	end

	def gem *args
	self.class.top_level_dependencies << args.first
	end

	# Only parse other gemfiles if they are from gems pending, otherwise ignore
	def eval_gemfile filename
	if filename.include? "gems/pending/Gemfile"
	gems_pending_gemfile = File.expand_path "../../gems/pending/Gemfile", __FILE__
	extra_gemfiles_to_eval << gems_pending_gemfile if File.exist? gems_pending_gemfile
	end
	end

	def group *types
	unless types.include? :development or types.include? :test
	yield if block_given?
	end
	end

	def dependencies
	[]
	end

	def source *args; end;
	end


	GemfileTopLevelDepParser.parse File.expand_path "../../Gemfile", __FILE__