Join two Vmware providers in ManageIQ

join_vmware_providers.rb

# GPLv2 License
# =====================
#
# Copyright © 2018 Robb Manes <robb.manes@gmail.com>
#
# join_vmware_providers is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# join_vmware_providers is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with join_vmware_providers. If not, see <http://www.gnu.org/licenses/>.
#
# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# cd /var/www/miq/vmdb/
# bundle exec rails runner join_vmware_providers.rb --source <SRC_PROVIDER> --dest <DEST_PROVIDER>

require 'trollop'
require 'logger'

module JoinVmwareProvider
  class Cli
    attr_accessor :options

    def parse(args, env = {})
      args.shift if args.first == "--"
      self.options = Trollop.options(args) do
        banner "" \
               "Usage: cd /var/www/miq/vmdb/" \
               "       bundle exec rails runner tools/join_provider_data.rb [options]"
        opt :source,      "Source",      :type => :string
        opt :destination, "Destination", :type => :string
        opt :verbose,     "Verbose",     :type => :boolean
      end

      self.options = options.delete_if { |_n, v| v.blank? }

      Trollop::die :source, "must specify source provider" unless self.options.key? :source
      Trollop::die :destination, "must specify destination provider" unless self.options.key? :destination

      self
    end

    def run
      ::JoinVmwareProvider::ProviderMerger.new(options).run
    end

    def self.run(args, env = {})
      new.parse(args, env).run
    end
  end

  class ProviderMerger
    def initialize(args = {})
      @logger = Logger.new(STDOUT)
      if args.verbose
        @logger.level = Logger::DEBUG
      else
        @logger.level = Logger::INFO
      end

      @src_ems = get_provider args.source
      @dst_ems = get_provider args.destination
      @logger.info("Using source provider #{@src_ems.name}:#{@src_ems.id} and destination provider #{@dst_ems.name}:#{@dst_ems.id}.")
    end

    def get_provider(provider_string)
      unless provider_string.scan(/\D/).empty?
        ems = ManageIQ::Providers::Vmware::InfraManager.find_by(:name => provider_string)
      else
        ems = ManageIQ::Providers::Vmware::InfraManager.find_by(:id => provider_string)
      end

      if ems.nil?
        @logger.error("Found no EMS for id or string #{provider_string}.")
        exit
      end

      @logger.info("Found provider #{ems.name} with id #{ems.id}.")
      ems
    end

    def run()
      merge_duplicate_vms
      merge_duplicate_hosts
      merge_duplicate_clusters
    end

    def merge_duplicate_vms
      src_vms = @src_ems.vms
      dst_vms = @dst_ems.vms
      @logger.debug("Found #{src_vms.count} VM's on source provider and #{dst_vms.count} VM's on destination provider.")

      @logger.info("Merging duplicate VM data...")

      dups = 0
      dst_vms.each do |dst_vm|
        src_vm = src_vms.find_by(:name => dst_vm.name)

        unless src_vm.nil?
          @logger.debug("Beginning VM merge of #{dst_vm.name}...")


          @logger.debug("Merging metrics for VM #{dst_vm.name}...")
          src_metrics = src_vm.metrics.all
          src_metrics.each do |metric|
            dst_vm.metrics << metric
          end

          @logger.debug("Merging tags for VM #{dst_vm.name}...")
          src_tags = src_vm.tags.all
          src_tags.each do |tag|
            dst_vm.tags << tag
          end

          @logger.debug("Updating EVM owner for VM #{dst_vm.name}...")
          dst_vm.update(:evm_owner => src_vm.evm_owner)

          @logger.debug("Merging groups for VM #{dst_vm.name}...")
          dst_vm.update(:miq_group => src_vm.miq_group)

          @logger.debug("Updating retirement settings for VM #{dst_vm.name}...")
          dst_vm.retirement_state = src_vm.retirement_state
          dst_vm.retires_on = src_vm.retires_on

          dst_vm.save
          dups = dups + 1
          @logger.debug("Finished merge for VM #{dst_vm.name}...")
        end
      end
      @logger.info("Merged #{dups} duplicate VM's.")
    end

    def merge_duplicate_hosts
      src_hosts = @src_ems.hosts
      dst_hosts = @dst_ems.hosts
      @logger.debug("Found #{src_hosts.count} hosts on source provider and #{dst_hosts.count} hosts on destination provider.")

      @logger.info("Merging duplicate hosts data...")

      dups = 0
      dst_hosts.each do |dst_vm|
        src_host = src_hosts.find_by(:name => dst_host.name)

        unless src_host.nil?
          @logger.debug("Beginning host merge of #{dst_host.name}")

          @logger.debug("Merging metrics for host #{dst_host.name}")
          src_metrics = src_host.metrics.all
          src_metrics.each do |metric|
            dst_host.metrics << metric
          end

          @logger.debug("Merging tags for host #{dst_host.name}")
          src_tags = src_host.tags.all
          src_tags.each do |tag|
            dst_host.tags << tag
          end

          dups = dups + 1
          @logger.debug("Finished merge for host #{dst_host.name}...")
        end
      end
      @logger.info("Merged #{dups} duplicate hosts.")
    end

    def merge_duplicate_clusters
      src_clusters = @src_ems.clusters
      dst_clusters = @dst_ems.clusters
      @logger.debug("Found #{src_clusters.count} clusters on source provider and #{dst_clusters.count} clusters on destination provider.")

      @logger.info("Merging duplicate cluster data...")

      dups = 0
      dst_clusters.each do |dst_cluster|
        src_cluster = src_clusters.find_by(:name => dst_cluster.name)

        unless src_cluster.nil?
          @logger.debug("Beginning cluster merge of #{dst_cluster.name}")

          @logger.debug("Merging metrics for cluster #{dst_cluster.name}")
          src_metrics = src_cluster.metrics.all
          src_metrics.each do |metric|
            dst_cluster.metrics << metric
          end

          @logger.debug("Merging tags for cluster #{dst_cluster.name}")
          src_tags = src_cluster.tags.all
          src_tags.each do |tag|
            dst_cluster.tags << tag
          end

          dups = dups + 1
          @logger.debug("Finished merge for cluster #{dst_cluster.name}...")
        end
      end
      @logger.info("Merged #{dups} duplicate clusters.")
    end
  end
end

JoinVmwareProvider::Cli.run(ARGV, ENV)

Rob, I am looking for a script to help with migrating from one vmware provider to another. I am migrating from our old infrastructure with windows vcenter 6.0 to a new VXrail with vcenter appliance 6.5 and I am trying to retain all the ownership and retirement info in CloudForms. Someone suggested this script might help, but I am struggling to understand it. If you have any further info, I would appreciate any guidance at all.

Hi @wddossett; the script should be placed in the manageiq directory and run as bundle exec rails runner join_vmware_providers.rb --source <SRC_PROVIDER> --dest <DEST_PROVIDER>, where you can either give the provider names or their ID's in MIQ. After that, it should tell you each step of the operation, merging specifically now metrics, tags, ownership, and retirement for each applicable item (VM's, hosts, clusters) between two VCenters of theoretically any versions. This is not very well tested at the moment, hence the disclaimer, and unsupported, but it has proven to work a few times for various individuals doing what you're doing. If it doesn't in some way, let me know and I'll see if I can address the issue here in the Gist.

EDIT: Concerning retirement states, it moves those, but not much else. Retirement dates might be off, I can see if that can be added, but metrics, ownership, and tags seem to work as expected. I'd have to look up each retirement attribute to move over individually, which might take me a bit.

Hi @robbmanes,
I am facing the below error, appreciate your kind looking at it:
I have already installed the required 'trollop' and 'logger' but still getting these errors.

[root@drmdcvmiq vmdb]# bundle exec rails runner join_vmware_providers.rb --source aeadsvcr01.wrm.ae  --dest drmdcvvc01.wrm.ae
Traceback (most recent call last):
        14: from bin/rails:4:in `<main>'
        13: from bin/rails:4:in `require'
        12: from /usr/local/lib/ruby/gems/2.5.0/gems/railties-5.1.7/lib/rails/commands.rb:16:in `<top (required)>'
        11: from /usr/local/lib/ruby/gems/2.5.0/gems/railties-5.1.7/lib/rails/command.rb:44:in `invoke'
        10: from /usr/local/lib/ruby/gems/2.5.0/gems/railties-5.1.7/lib/rails/command/base.rb:63:in `perform'
         9: from /usr/local/lib/ruby/gems/2.5.0/gems/thor-1.0.1/lib/thor.rb:392:in `dispatch'
         8: from /usr/local/lib/ruby/gems/2.5.0/gems/thor-1.0.1/lib/thor/invocation.rb:127:in `invoke_command'
         7: from /usr/local/lib/ruby/gems/2.5.0/gems/thor-1.0.1/lib/thor/command.rb:27:in `run'
         6: from /usr/local/lib/ruby/gems/2.5.0/gems/railties-5.1.7/lib/rails/commands/runner/runner_command.rb:34:in `perform'
         5: from /usr/local/lib/ruby/gems/2.5.0/gems/railties-5.1.7/lib/rails/commands/runner/runner_command.rb:34:in `load'
         4: from join_vmware_providers.rb:31:in `<top (required)>'
         3: from /usr/local/lib/ruby/gems/2.5.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `require'
         2: from /usr/local/lib/ruby/gems/2.5.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:258:in `load_dependency'
         1: from /usr/local/lib/ruby/gems/2.5.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `block in require'
/usr/local/lib/ruby/gems/2.5.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `require': cannot load such file -- trollop (LoadError)
[root@drmdcvmiq vmdb]#

I haven't used this script since the beginning of 2019, and don't know if something changed in CloudForms that might cause a dependency error.

From the output, it looks like perhaps CloudForms doesn't use trollop anymore and the gem isn't present:

/usr/local/lib/ruby/gems/2.5.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `require': cannot load such file -- trollop (LoadError)

You might be able to gem install trollop to fix that, but I am not sure of the level of support you can expect from such a thing in terms of support. Otherwise I'd have to rewrite the script without trollop, using some other command-line manager, and I haven't been working with Ruby for some time so I'd have to dive into what's A) current B) included on CloudForms appliances now.

The trollop gem was switched out for the optimist gem, because, rightfully so, it's pretty offensive. Anything on CFME >= 5.0 won't work with the above script.

Changelog in Red Hat documentation is available here.

I wrote this for CFME 4.5 and haven't kept it updated, so it might be possible it doesn't work with the intended consequences anymore as well. You're welcome to try and switch out trollop for optimist where applicable, and if someone verifies that that's all that needs to be done (I no longer work on CloudForms) then I can update it here.