bosh release copy script

release_copy.rb

#!/usr/bin/env ruby

require "cli"
require "cli/release"
require "cli/blob_manager"
require "blobstore_client"
require "cli/versions/versions_index"

require 'optparse'

BEGIN {
  require 'net/http'
  Net::HTTP.module_eval do
    alias_method '__initialize__', 'initialize'
    def initialize(*args,&block)
      __initialize__(*args, &block)
      ensure
      @debug_output = $stderr if ENV['HTTP_DEBUG']
    end
  end
}

DEFAULT_INDEX_NAME = "blobs.yml"

options = {}
opt_parser = OptionParser.new do |opts|
  opts.banner = "Usage: #{__FILE__} [options]"

  opts.on("-r RELEASE_DIR", "--release RELEASE_DIR", "Release to copy blobs from") do |r|
    options[:release] = r
  end

  opts.on("-p PRIVATE_FILE", "--private-file PRIVATE_FILE", "Private file with credentials for the target blobstore. Default ./private.yml") do |pf|
    options[:private] = pf
  end

  opts.on("-h", "--help", "Display usage help") do |h|
    options[:help] = h
  end
end
opt_parser.parse!


def create_blobstore_client(config)
  blobstore_cfg = config["blobstore"]

  provider = blobstore_cfg["provider"]
  opts = blobstore_cfg[provider]
  Bosh::Blobstore::Client.safe_create(provider, opts)
end

def copy_blob(id, sha, source_blobstore, target_blobstore, desc)
  if target_blobstore.exists?(id)
    warn("[WARNING] Object with id [#{id}] already exists in target blobstore -> skip copying")
    return id
  end

  unless source_blobstore.exists?(id)
    err("[ERROR] Object with id [#{id}] does not exists in src blobstore!")
    return id
  end


  puts "need to copy the object [#{id}]"

  tmp_fname = download_blob(id, sha, source_blobstore, desc)
  tmp_file = File.new(tmp_fname, "r")

  uploaded_id = target_blobstore.create(tmp_file, id)
  tmp_file.close
  puts "uploaded object with id #{uploaded_id}"

  uploaded_id

end

def download_blob(object_id, sha, blobstore, desc = "blob")
      tmp_file = File.open(File.join(Dir.mktmpdir, "bosh-blob"), "w")
      puts "Downloading `#{desc}' with id #{object_id} to #{tmp_file.path}"
      blobstore.get(object_id, tmp_file)
      tmp_file.close

      if file_checksum(tmp_file.path) != sha
        obj = desc ? desc : object_id
        err("Checksum mismatch for downloaded blob `#{obj}'")
      end

      tmp_file.path
end

def file_checksum(path)
  Digest::SHA1.file(path).hexdigest
end

def copy_release_blobs(release, target_blobstore)
  index_file = File.join(release.dir, "config", DEFAULT_INDEX_NAME)

  index = load_yaml_file(index_file)

  index.each_pair do |path, entry|
    puts "Processing blob: #{path}"

    copy_blob(entry["object_id"], entry["sha"], release.blobstore, target_blobstore, path)
  end
end

def get_final_release_name(release)
  final_config_file = File.join(release.dir, "config", "final.yml")
  config = load_yaml_file(final_config_file)

  config["final_name"]
end

def get_latest_release_yaml(release)
  releases_idx = Bosh::Cli::Versions::VersionsIndex.new(File.join(release.dir, "releases"))
  release_name = get_final_release_name(release)
  release_fname = release_name + "-" + releases_idx.version_strings.last + ".yml"

  yaml_name = File.join(release.dir, "releases", release_fname)
  puts "Using release file: #{yaml_name}"

  release_yaml = load_yaml_file(yaml_name)
end


def copy_license(release, target_blobstore)

  release_yaml = get_latest_release_yaml(release)

  # copy licence
  lic = release_yaml["license"]

  lic_idx = Bosh::Cli::Versions::VersionsIndex.new(File.join(release.dir, ".final_builds", "license"))

  lic_entry = lic_idx[lic_idx.find_key_by_version(lic["version"])]
  copy_blob(lic_entry["blobstore_id"], lic_entry["sha1"], release.blobstore, target_blobstore, "license ")
end



def copy_release_packages_and_jobs(release, target_blobstore)

  release_yaml = get_latest_release_yaml(release)

  # copy packages
  release_yaml["packages"].each do |pkg|
    pkg_name = pkg["name"]
    puts "Processing package: #{pkg_name}"

    pkg_idx = Bosh::Cli::Versions::VersionsIndex.new(File.join(release.dir, ".final_builds", "packages", pkg["name"]))
    pkg_entry = pkg_idx[pkg_idx.find_key_by_version(pkg["version"])]

    copy_blob(pkg_entry["blobstore_id"], pkg_entry["sha1"], release.blobstore, target_blobstore, "package " + pkg_name)
  end

  # copy blobs
  release_yaml["jobs"].each do |job|
    job_name = job["name"]
    puts "Processing job: #{job_name}"

    job_idx = Bosh::Cli::Versions::VersionsIndex.new(File.join(release.dir, ".final_builds", "jobs", job["name"]))
    job_entry = job_idx[job_idx.find_key_by_version(job["version"])]

    copy_blob(job_entry["blobstore_id"], job_entry["sha1"], release.blobstore, target_blobstore, "job " + job_name)
  end

end

##
#
# main
#
##

if !options[:release]
  puts "Have to specify a RELEASE_DIR"
  puts opt_parser
  exit -1
end

target_blobstore_config = options[:private] ? options[:private] : File.join(Dir.pwd, "private.yml")
unless File.exist?(target_blobstore_config)
  puts "Could not find private file with configuration for the target blobstore. #{target_blobstore_config} does not exist."
  exit -1
end

release = Bosh::Cli::Release.new(options[:release])
src_blobstore = release.blobstore

config = load_yaml_file(target_blobstore_config)
target_blobstore = create_blobstore_client(config)

copy_release_blobs(release, target_blobstore)
copy_release_packages_and_jobs(release, target_blobstore)
copy_license(release, target_blobstore)