DynamoDB Parallel Scan

scan.rb

require 'aws-sdk'
require 'pry-byebug'

class MobyEvent
  SCAN_WORKERS = 4

  class << self
    def client
      @dynamodb ||= Aws::DynamoDB::Client.new(
        region:             'us-east-1',
        access_key_id:      '...',
        secret_access_key:  '...'
      )
    end

    def scan(filter_expression, expression_attribute_values)
      items = Queue.new

      SCAN_WORKERS.times.map do |seg|
        Thread.new do
          resp = client.scan(
            table_name: 'moby_events',
            limit: 1,
            total_segments: SCAN_WORKERS,
            segment: seg,
            expression_attribute_values: expression_attribute_values,
            filter_expression: filter_expression
          )

          items.push resp.items
        end

      end.each &:join

      items
    end
  end
end

started_at = Time.now

items = MobyEvent.scan('attempts > :s', ':s' => 0)

puts "Completed in: #{(Time.now - started_at) * 1000} ms"

puts items.size

while !items.empty?
  puts items.pop.inspect
end

puts 'done'