wcdd_etl.rb

#!/usr/bin/ruby 

require 'riak'
require 'csv'

begin
  case ARGV.size
  when 1 then   port = ARGV[0].to_i
  when 0 then   port = 8097
  else   raise  ArgumentException
  end
  raise ArgumentError if port == 0 or port > 65535
rescue NoMethodError, ArgumentError
  puts "Error: Invalid Arguments Specified"
  puts "Usage: #{$0} [port number, must be between 1 and 65535]"
  exit(1)
end

client        = Riak::Client.new(:port => port)

wcdd_dir      = "sample_data/Windy-City-DB-Dataset/data"
wcdd_json     = {}
bucket_names  = ["answers", "comments", "posts", "users"]

Dir.glob("#{wcdd_dir}/*.json").each do |json_file|
  wcdd_json.merge! JSON.parse(File.read json_file)
end

bucket_names.each do |name|
  bucket  = client["wcdd_#{name}"]

  wcdd_json[name].each do |entry|
    begin
      key       = bucket.get entry["Id"]
    rescue StandardError
      key       = bucket.new entry["Id"]
      key.data  = entry

      begin
        key.store
      rescue => e
        puts "Failed to save '#{name}' key ID: #{entry["Id"]}; #{e}"
      end
    end # double rescue
  end

end