NickLaMuro/README.md Secret

## README.md

      
    Raw
  

              README.md
            
          
    Bugzilla 1648412 replication/benchmark

This repo provides two scripts:

bz_1648412_replication_database_seed.rb
bz_1648412_replication_benchmark.rb

Usage

This is instructions for not only using the script, but seeing the differences
that the patch from ManageIQ/manageiq-ui-classic#5283
is applied.
These steps should be run either from a dev environment of ManageIQ/manageiq or
a running MIQ/CFME appliance in the /var/www/miq/vmdb directory.


Clean out your database by running the following (skip if you are on a fresh
appliance):
$ bin/rake db:drop db:create db:migrate db:seed


Run the database seed script
$ bin/rails r bz_1648412_replication_database_seed.rb


Run the benchmark script and make note of the timings of the last two
requests:
$ bin/rails r bz_1648412_replication_benchmark.rb


Update the product/views/VmOrTemplate.yaml in manageiq-ui-classic to
remove the compliances table from the include_for_find in the report.
Simplest way is probably to just edit your copy by doing:
$ export EDITOR=vim # or whatever you like
$ bundle open manageiq-ui-classic
(see https://github.com/ManageIQ/manageiq-ui-classic/pull/5283/files for
what needs to change)


Re-run step 3, taking note of the timing improvements.


## bz_1648412_replication_benchmark.rb
########## SETUP ###########

# Supress ActiveRecord SQL statements from being logged
ActiveRecord::Base.logger = Logger.new(STDOUT).tap {|l| l.level = Logger::INFO }

# Supress output from ActionView
ActionView::Base.logger   = Logger.new(File::NULL)

# no-op noisy method from ui-classic
ApplicationController
class ApplicationController < ActionController::Base; def get_data_size(hash); end; end


# Load Rails helper methods
Rails.application.load_console
Rails.env = ENV["RAILS_ENV"]
include Rails::ConsoleMethods
# UiConstants # uncomment this if you are using a older version of MIQ/CFME
MiqUiWorker.preload_for_console


# Small active record extension to count rows in the PostgreSQLAdapter.
#
# Pulled from `ManageIQPerformance::Middlewares::ActiveRecordQueries`.
class SQLRowCounter
  include Singleton

  attr_reader :row_count

  # Set the active_support listener and
  def self.listen
    ActiveSupport::Notifications.subscribe("record_count.active_record", instance)
  end

  def self.reset &block
    instance.reset &block
  end

  def initialize
    reset
  end

  # What is called by ActiveSupport::Notifications
  def call(name, _, _, _, payload)
    if name == "record_count.active_record"
      @row_count += payload[:record_count]
    end
  end

  # Reset the count to zero.
  #
  # If a block is given, pass the @row_count before resetting
  def reset
    yield @row_count if block_given?
    @row_count = 0
  end

  # Run reset, appending the Row count into the 'Completed 200 OK...' string
  module PostActionPrinter
    extend ActiveSupport::Concern
    module ClassMethods
      def log_process_action(payload)
        super.tap do |messages|
          ::SQLRowCounter.reset { |row_count| messages << "Rows: #{row_count}" }
        end
      end
    end
  end

  module AdapterNotifiers
    # patch `#exec_query` so we include a ActiveSupport::Notifications call for
    # the number of rows fetched.
    def exec_query(*args)
      payload     = {}
      label       = "record_count.active_record"
      message_bus = ActiveSupport::Notifications.instrumenter

      message_bus.instrument label, payload do
        super.tap { |records| payload[:record_count] = records.length }
      end
    end
  end
end

# Install the monkey patches and listener
class ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
  prepend SQLRowCounter::AdapterNotifiers
end
class ActionController::Base
  include SQLRowCounter::PostActionPrinter
end
SQLRowCounter.listen


# Helper for setting the csrf_token
TOKEN_REGEXP = /.*csrf-token.*content="(?<CSRF_TOKEN>[^"]*)"/
def post_headers
  @csrf_token_headers ||= {
    :"X-CSRF-Token" => app.response.body.match(TOKEN_REGEXP)[:CSRF_TOKEN]
  }
end


########## REQUESTS #########


# Login
app.post "/dashboard/authenticate", :params => { :user_name   => "admin", :user_password => "smartvm" }

# Equivalent to doing:
#
#   - Visit `Compute -> Infrastructure -> Virtual Machines`
#   - Clicking the next page pagination button
#
app.get  "/vm_infra/explorer"
app.post "/vm_infra/report_data", :headers => post_headers,
                                  :params  => { :page => 1, :ppsetting => 20 }
app.post "/vm_infra/report_data", :headers => post_headers,
                                  :params  => { :page => 2, :ppsetting => 20 }

## bz_1648412_replication_database_seed.rb
# Database seed script for replicating performance issues found in the following bugzilla ticket:
#
#   https://bugzilla.redhat.com/show_bug.cgi?id=1648412
#

# Create Zone
Zone.last || Zone.seed
zone = Zone.last

# Create Storage records
storage_records = []
if Storage.count.positive?
  puts "Storage records created!"
  storage_records = Storage.all
else
  puts "Creating Storage records..."
  19.times do |i|
    id      = i + 1
    ref_obj = VimString.new("datastore-#{id}", ":Datastore", ":ManagedObjectReference")
    storage_data = {
      :name                          => "storage_nfs_200_TB_#{'%03d' % id}",
      :store_type                    => "NFS",
      :free_space                    => 100.terabytes,
      :total_space                   => 200.terabytes,
      :uncommitted                   => 1.gigabyte,
      :multiplehostaccess            => 1,
      :last_perf_capture_on          => Time.now,
      :location                      => "1234567-abcde#{'%03d' % id}",
      :directory_hierarchy_supported => true,
      :thin_provisioning_supported   => true,
      :raw_disk_mappings_supported   => false,
      :master                        => false,
      :ems_ref_obj                   => ref_obj,
      :ems_ref                       => "datastore-#{id}"
    }
    storage_records << Storage.create(storage_data)
  end
end

# Create EMS records
ems_records = []
if ManageIQ::Providers::Vmware::InfraManager.count.positive?
  puts "EMS records created!"
  ems_records = ManageIQ::Providers::Vmware::InfraManager.all
else
  puts "Creating EMS records..."
  14.times do |i|
    id = i + 1
    ems_data = {
      :name             => "ems_#{'%03d' % id}",
      :hostname         => "ems-#{'%03d' % id}",
      :ipaddress        => "10.#{'%03d' % id}.0.0",
      :guid             => SecureRandom.uuid,
      :zone             => zone,
      :storage_profiles => []
    }
    ems_records << ManageIQ::Providers::Vmware::InfraManager.create!(ems_data)
  end
end

ems_records.each {|r| puts r.inspect; puts "  #{r.ipaddress}"}

# Create EMS Cluster records
cluster_records = []
if EmsCluster.count.positive?
  puts "Cluster records created!"
  cluster_records = EmsCluster.all
else
  puts "Creating Cluster records..."

  build_ems_data = lambda do |ems, i|
    id           = i + 1
    ems_ref_obj  = VimString.new("domain-#{id}", ":ClusterComputeResource", ":ManagedObjectReference")
    cluster_data = {
      :name                    => "EMS_CLUSTER_DOMAIN_#{id}",
      :ems_id                  => ems.id,
      :uid_ems                 => ems_ref_obj.to_s,
      :ha_enabled              => true,
      :ha_admit_control        => true,
      :drs_enabled             => true,
      :drs_automation_level    => "fullyAutomated",
      :drs_migration_threshold => 3,
      :effective_cpu           => 1000000,
      :effective_memory        => 64.gigabytes,
      :ems_ref_obj             => ems_ref_obj,
      :ems_ref                 => ems_ref_obj.to_s
    }
  end

  ems_records.each_with_index do |ems, i|
    cluster_records << EmsCluster.create(build_ems_data.call(ems, i))
  end

  # Add 3 more randomly dispersed clusters
  (ems_records.size..(ems_records.size + 3)).each do |i|
    cluster_records << EmsCluster.create(build_ems_data.call(ems_records.sample, i))
  end
end

# Create Tags
tag_records = []
if Tag.where("name LIKE '/namespace%'").count.positive?
  puts "Tag records created!"
  tag_records = Tag.where("name LIKE '/namespace%'").to_a
else
  puts "Creating Tag Records..."
  100.times do |i|
    id = i + 1
    tag_data = { :name => "/namespace/cat/tag_#{'%03d' % id}" }
    tag_records << Tag.create(tag_data)
  end
end

# Create Host Records
host_records = []
if Host.count.positive?
  puts "Host records created!"
  host_records = Host.all
else
  puts "Creating Host records..."
  22.times do |i|
    id        = i + 1
    cluster   = cluster_records.sample
    ems       = cluster.ext_management_system
    host_data = {
      :name                  => "host_#{'%03d' % id}",
      :hostname              => "host-#{'%03d' % id}",
      :vmm_vendor            => "vmware",
      :ipaddress             => ems.ipaddress.sub(/\.0$/, '%03d' % id),
      :user_assigned_os      => "linux_generic",
      :power_state           => "on",
      :ems_cluster           => cluster,
      :ext_management_system => ems,
      :tags                  => tag_records.sample(75)
    }
    host_records << ManageIQ::Providers::Vmware::InfraManager::Host.create(host_data)
  end
end

# Create MiqTemplate Records
template_records = []
if MiqTemplate.count.positive?
  puts "MiqTemplate records created!"
else
  puts "Creating MiqTemplate records..."
  40.times do |i|
    id   = i + 1
    host = host_records.sample
    template_data = {
      :name                  => "vm_#{'%05d' % id}",
      :location              => "[storage] vm_#{'%05d' % id}/vm_#{'%05d' % id}.vmx",
      :uid_ems               => SecureRandom.uuid,
      :vendor                => "vmware",
      :raw_power_state       => "poweredOn",
      :host                  => host,
      :ems_id                => host.ems_id,
      :storage               => storage_records.sample,
      :tags                  => tag_records,
      :template              => true
    }
    vm = ManageIQ::Providers::Vmware::InfraManager::Template.create(template_data)
    hw = Hardware.create(:miq_template => vm)
    hw = OperatingSystem.create(:miq_template => vm)

    template_records << vm
  end

  puts "Creating Compliance records..."
  compliance_runs  = [1,   1,   1,   1,   1,   1,   1,   1,   1,   1,
                      52,  70,  70,  26,  22,  22,  44,  14,  22,  1,
                      3,   14,  42,  12, 122,  70,  21,  70,  54,  185,
                      97,  56,  11,  92, 15, 132, 53, 35, 49, 120]
  compliance_vals  = [true, true, true, true, true, true, true, false]
  auth_status_vals = [nil, "Valid", "None", "Incomplete", "ERROR", "Unreachable", "Invalid"]
  template_records.each do |template|
    # compliances
    compliance_runs.shift.times do
      Compliance.create(
        :resource           => template,
        :compliant          => compliance_vals.sample,
        :timestamp          => DateTime.current - rand(120).days,
        :updated_on         => DateTime.current,
        :event_type         => 'string',
        :compliance_details => [],
      )
    end
  end
end
	########## SETUP ###########

	# Supress ActiveRecord SQL statements from being logged
	ActiveRecord::Base.logger = Logger.new(STDOUT).tap {\|l\| l.level = Logger::INFO }

	# Supress output from ActionView
	ActionView::Base.logger = Logger.new(File::NULL)

	# no-op noisy method from ui-classic
	ApplicationController
	class ApplicationController < ActionController::Base; def get_data_size(hash); end; end


	# Load Rails helper methods
	Rails.application.load_console
	Rails.env = ENV["RAILS_ENV"]
	include Rails::ConsoleMethods
	# UiConstants # uncomment this if you are using a older version of MIQ/CFME
	MiqUiWorker.preload_for_console


	# Small active record extension to count rows in the PostgreSQLAdapter.
	#
	# Pulled from `ManageIQPerformance::Middlewares::ActiveRecordQueries`.
	class SQLRowCounter
	include Singleton

	attr_reader :row_count

	# Set the active_support listener and
	def self.listen
	ActiveSupport::Notifications.subscribe("record_count.active_record", instance)
	end

	def self.reset &block
	instance.reset &block
	end

	def initialize
	reset
	end

	# What is called by ActiveSupport::Notifications
	def call(name, _, _, _, payload)
	if name == "record_count.active_record"
	@row_count += payload[:record_count]
	end
	end

	# Reset the count to zero.
	#
	# If a block is given, pass the @row_count before resetting
	def reset
	yield @row_count if block_given?
	@row_count = 0
	end

	# Run reset, appending the Row count into the 'Completed 200 OK...' string
	module PostActionPrinter
	extend ActiveSupport::Concern
	module ClassMethods
	def log_process_action(payload)
	super.tap do \|messages\|
	::SQLRowCounter.reset { \|row_count\| messages << "Rows: #{row_count}" }
	end
	end
	end
	end

	module AdapterNotifiers
	# patch `#exec_query` so we include a ActiveSupport::Notifications call for
	# the number of rows fetched.
	def exec_query(*args)
	payload = {}
	label = "record_count.active_record"
	message_bus = ActiveSupport::Notifications.instrumenter

	message_bus.instrument label, payload do
	super.tap { \|records\| payload[:record_count] = records.length }
	end
	end
	end
	end

	# Install the monkey patches and listener
	class ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
	prepend SQLRowCounter::AdapterNotifiers
	end
	class ActionController::Base
	include SQLRowCounter::PostActionPrinter
	end
	SQLRowCounter.listen


	# Helper for setting the csrf_token
	TOKEN_REGEXP = /.csrf-token.content="(?<CSRF_TOKEN>[^"]*)"/
	def post_headers
	@csrf_token_headers \|\|= {
	:"X-CSRF-Token" => app.response.body.match(TOKEN_REGEXP)[:CSRF_TOKEN]
	}
	end




	########## REQUESTS #########



	# Login
	app.post "/dashboard/authenticate", :params => { :user_name => "admin", :user_password => "smartvm" }

	# Equivalent to doing:
	#
	# - Visit `Compute -> Infrastructure -> Virtual Machines`
	# - Clicking the next page pagination button
	#
	app.get "/vm_infra/explorer"
	app.post "/vm_infra/report_data", :headers => post_headers,
	:params => { :page => 1, :ppsetting => 20 }
	app.post "/vm_infra/report_data", :headers => post_headers,
	:params => { :page => 2, :ppsetting => 20 }
	# Database seed script for replicating performance issues found in the following bugzilla ticket:
	#
	# https://bugzilla.redhat.com/show_bug.cgi?id=1648412
	#

	# Create Zone
	Zone.last \|\| Zone.seed
	zone = Zone.last

	# Create Storage records
	storage_records = []
	if Storage.count.positive?
	puts "Storage records created!"
	storage_records = Storage.all
	else
	puts "Creating Storage records..."
	19.times do \|i\|
	id = i + 1
	ref_obj = VimString.new("datastore-#{id}", ":Datastore", ":ManagedObjectReference")
	storage_data = {
	:name => "storage_nfs_200_TB_#{'%03d' % id}",
	:store_type => "NFS",
	:free_space => 100.terabytes,
	:total_space => 200.terabytes,
	:uncommitted => 1.gigabyte,
	:multiplehostaccess => 1,
	:last_perf_capture_on => Time.now,
	:location => "1234567-abcde#{'%03d' % id}",
	:directory_hierarchy_supported => true,
	:thin_provisioning_supported => true,
	:raw_disk_mappings_supported => false,
	:master => false,
	:ems_ref_obj => ref_obj,
	:ems_ref => "datastore-#{id}"
	}
	storage_records << Storage.create(storage_data)
	end
	end

	# Create EMS records
	ems_records = []
	if ManageIQ::Providers::Vmware::InfraManager.count.positive?
	puts "EMS records created!"
	ems_records = ManageIQ::Providers::Vmware::InfraManager.all
	else
	puts "Creating EMS records..."
	14.times do \|i\|
	id = i + 1
	ems_data = {
	:name => "ems_#{'%03d' % id}",
	:hostname => "ems-#{'%03d' % id}",
	:ipaddress => "10.#{'%03d' % id}.0.0",
	:guid => SecureRandom.uuid,
	:zone => zone,
	:storage_profiles => []
	}
	ems_records << ManageIQ::Providers::Vmware::InfraManager.create!(ems_data)
	end
	end

	ems_records.each {\|r\| puts r.inspect; puts " #{r.ipaddress}"}

	# Create EMS Cluster records
	cluster_records = []
	if EmsCluster.count.positive?
	puts "Cluster records created!"
	cluster_records = EmsCluster.all
	else
	puts "Creating Cluster records..."

	build_ems_data = lambda do \|ems, i\|
	id = i + 1
	ems_ref_obj = VimString.new("domain-#{id}", ":ClusterComputeResource", ":ManagedObjectReference")
	cluster_data = {
	:name => "EMS_CLUSTER_DOMAIN_#{id}",
	:ems_id => ems.id,
	:uid_ems => ems_ref_obj.to_s,
	:ha_enabled => true,
	:ha_admit_control => true,
	:drs_enabled => true,
	:drs_automation_level => "fullyAutomated",
	:drs_migration_threshold => 3,
	:effective_cpu => 1000000,
	:effective_memory => 64.gigabytes,
	:ems_ref_obj => ems_ref_obj,
	:ems_ref => ems_ref_obj.to_s
	}
	end

	ems_records.each_with_index do \|ems, i\|
	cluster_records << EmsCluster.create(build_ems_data.call(ems, i))
	end

	# Add 3 more randomly dispersed clusters
	(ems_records.size..(ems_records.size + 3)).each do \|i\|
	cluster_records << EmsCluster.create(build_ems_data.call(ems_records.sample, i))
	end
	end

	# Create Tags
	tag_records = []
	if Tag.where("name LIKE '/namespace%'").count.positive?
	puts "Tag records created!"
	tag_records = Tag.where("name LIKE '/namespace%'").to_a
	else
	puts "Creating Tag Records..."
	100.times do \|i\|
	id = i + 1
	tag_data = { :name => "/namespace/cat/tag_#{'%03d' % id}" }
	tag_records << Tag.create(tag_data)
	end
	end

	# Create Host Records
	host_records = []
	if Host.count.positive?
	puts "Host records created!"
	host_records = Host.all
	else
	puts "Creating Host records..."
	22.times do \|i\|
	id = i + 1
	cluster = cluster_records.sample
	ems = cluster.ext_management_system
	host_data = {
	:name => "host_#{'%03d' % id}",
	:hostname => "host-#{'%03d' % id}",
	:vmm_vendor => "vmware",
	:ipaddress => ems.ipaddress.sub(/\.0$/, '%03d' % id),
	:user_assigned_os => "linux_generic",
	:power_state => "on",
	:ems_cluster => cluster,
	:ext_management_system => ems,
	:tags => tag_records.sample(75)
	}
	host_records << ManageIQ::Providers::Vmware::InfraManager::Host.create(host_data)
	end
	end

	# Create MiqTemplate Records
	template_records = []
	if MiqTemplate.count.positive?
	puts "MiqTemplate records created!"
	else
	puts "Creating MiqTemplate records..."
	40.times do \|i\|
	id = i + 1
	host = host_records.sample
	template_data = {
	:name => "vm_#{'%05d' % id}",
	:location => "[storage] vm_#{'%05d' % id}/vm_#{'%05d' % id}.vmx",
	:uid_ems => SecureRandom.uuid,
	:vendor => "vmware",
	:raw_power_state => "poweredOn",
	:host => host,
	:ems_id => host.ems_id,
	:storage => storage_records.sample,
	:tags => tag_records,
	:template => true
	}
	vm = ManageIQ::Providers::Vmware::InfraManager::Template.create(template_data)
	hw = Hardware.create(:miq_template => vm)
	hw = OperatingSystem.create(:miq_template => vm)

	template_records << vm
	end

	puts "Creating Compliance records..."
	compliance_runs = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	52, 70, 70, 26, 22, 22, 44, 14, 22, 1,
	3, 14, 42, 12, 122, 70, 21, 70, 54, 185,
	97, 56, 11, 92, 15, 132, 53, 35, 49, 120]
	compliance_vals = [true, true, true, true, true, true, true, false]
	auth_status_vals = [nil, "Valid", "None", "Incomplete", "ERROR", "Unreachable", "Invalid"]
	template_records.each do \|template\|
	# compliances
	compliance_runs.shift.times do
	Compliance.create(
	:resource => template,
	:compliant => compliance_vals.sample,
	:timestamp => DateTime.current - rand(120).days,
	:updated_on => DateTime.current,
	:event_type => 'string',
	:compliance_details => [],
	)
	end
	end
	end