openstack_swift_batch_upload.rb

# uploads and retrieves route data to/from OpenStack Swift
class SwiftRouteUpload
  RC_ROUTES = 'runningcoach_routes'.freeze
  AUTH_METHOD = 'password'.freeze
  AUTH_URL = 'https://cloudstorage.flow.ch/auth/v1.0'.freeze
  SERVICE_TYPE = 'object-store'.freeze
  META_DATA = { metadata: {}, content_type: 'application/json' }.freeze
  COUNT = 100

  def initialize(options)
    @options = options
  end

  def retrieve_object(training_id, postfix = nil)
    name = name_for(training_id, postfix)
    obj = container(training_id).object(name)
    obj.data
  end

  def create_object(training_id, data, postfix = nil)
    name = name_for(training_id, postfix)
    container(training_id).create_object(name, META_DATA, data)
    object_exists?(training_id, postfix)
  end

  def delete_object(training_id, postfix = nil)
    name = name_for(training_id, postfix)
    container(training_id).delete_object(name)
  end

  def object_exists?(training_id, postfix = nil)
    return false unless container_exists?(training_id)
    name = name_for(training_id, postfix)
    container(training_id).object_exists?(name)
  end

  def create_container(name)
    connection.create_container(name)
  end

  def delete_container(name)
    connection.delete_container(name)
  end

  private

  def container_exists?(training_id)
    connection.container_exists?("runningcoach_routes_#{training_id % COUNT}")
  end

  def container(training_id)
    connection.container("runningcoach_routes_#{training_id % COUNT}")
  end

  def connection
    @connection ||= OpenStack::Connection.create(
      username: @options['username'],
      api_key: @options['password'],
      auth_method: AUTH_METHOD,
      auth_url: AUTH_URL,
      service_type: SERVICE_TYPE
    )
  end

  def name_for(training_id, postfix)
    name = training_id.to_s
    name += "-#{postfix}" if postfix
    name
  end
end

uploader = SwiftRouteUpload.new(Rails.application.secrets.openstack_swift)

result = Dir["#{Rails.root}/public/uploads/routes/*"].map do |filename|
  next unless File.directory?(filename)

  Thread.new do

    Dir.entries(filename).each do |entry|
      next if ['.', '..'].include?(entry)

      path = "#{filename}/#{entry}"

      unless File.readable?(path)
        msg = "File is not readable: #{path}"
        p msg
        csv << "#{msg}\n" && next
      end

      name = File.basename(filename)
      name = name.split('-')

      if uploader.object_exists?(name[0].to_i, name[1])
        msg = "File already exists: #{name.join('-')}"
        p msg
        csv << "#{msg}\n" && next
      end

      data = File.read(path)

      if uploader.create_object(name[0].to_i, data, name[1])
        msg = "File was saved: #{name.join('-')}"
        p msg
        csv << "#{msg}\n"
      end

    end

  end

end

result.compact.map(&:value)