AWS classes to support granular archival

aws_s3.rb

module Starfish
  module AwsS3
    
    # S3Archive initializes and provides archival-oriented access to an S3 bucket.
    class S3Archive
    
      def initialize
        @config ||= YAML.load_file("#{File.expand_path('~') + '/.ec2' + '/aws-secret.yml'}")
        AWS::S3::Base.establish_connection!(
                  :access_key_id     => @config['aws']['access_key'], 
                  :secret_access_key => @config[['aws']['secret_access_key']
                )
      end
      
      def bucket(name, create=false)
        AWS::S3::Bucket.find(name) or AWS::S3::Bucket.create(name)
      end
      
      # Accepts a string or IO object. Compresses string data with zlib deflate.
      # Zlib::GzipReader generates an IO object compatible with this method.
      def put(bucket_name, key, data, opts={})
        options = {:content_type => 'binary/octet-stream'}.merge(opts)
        data = StringIO.new(Zlib::Deflate::deflate(data)) if data.class == String
        AWS::S3::S3Object.store(key, data, bucket_name, options)
      end

      # Accepts a string or IO object. Expands string data with zlib inflate.
      # Zlib::GzipWriter generates an IO object compatible with this method.
      def get(bucket_name, key, io=nil, &block)
        if io.respond_to?(:write)
          AWS::S3::S3Object.stream(key, bucket_name) do |chunk|
            io.write chunk
          end
        elsif block
          AWS::S3::S3Object.stream(key, bucket_name, {}, &block)
        else
          Zlib::Inflate::inflate(AWS::S3::S3Object.value(key, bucket_name))
        end
      end
    end
  end
end

aws_secret.yml

# Sign up for AWS and get your keys: http://aws.amazon.com/
# Keep your secrets out of the project repo. :)
aws:
  access_key: XXXXXXXXXXXXXXX
  secret_access_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
  account: XXXXXXXXXXXX

sqs_queue.rb

module Starfish
  module Pipeline
    
    # SqsQueue initializes and provides basic access to an AWS SQS queue.
    class SqsQueue
      attr_reader :queue, :name, :sqs
    
      def initialize(name)
        @config = YAML.load_file("#{File.expand_path('~') + '/.ec2' + '/aws-secret.yml'}")
        @sqs = RightAws::SqsGen2.new(
              @config['aws']['access_key'], 
              @config['aws']['secret_access_key'],
              {:logger => Rails.logger})      
    
        @name = name
        @queue = @sqs.queue(@name, true, "1")
      end
    
      # This method enqueues a message, assumed to be a Ruby hash object,
      # which is serialized to JSON.
      def send(message)
        @queue.send_message(JSON[message])
      end
    
      # This method both receives and deletes the message, assumed to be a JSON
      # object, and converts to a Ruby hash object.
      def pop
        if message = @queue.pop
          JSON[message.to_s]
        end
      end

      # This method peeks at a message and triggers the visibility timeout,
      # but doesn't remove/delete the message.
      def receive
        if message = @queue.receive
          JSON[message.to_s]
        end
      end
    
    end
  end
end