Skip to content

Instantly share code, notes, and snippets.

Avatar

Arfon Smith arfon

View GitHub Profile
@arfon
arfon / S3 uploader
Created Dec 10, 2009
Upload all files in directory to S3
View S3 uploader
require "rubygems"
require "net/http"
require "uri"
require "aws/s3"
AWS::S3::Base.establish_connection!(
:access_key_id => 'ACCESS KEY',
:secret_access_key => 'SECRET KEY'
)
View gist:430717
def new_method
puts "YEAH"
end
@arfon
arfon / create_table_composite_primary.rb
Created Jan 21, 2012
AWS DynamoDB table creation (composite primary_key)
View create_table_composite_primary.rb
require 'aws-sdk'
AWS_ACCESS_KEY = 'your_aws_access_key'
AWS_SECRET_KEY = 'your_aws_secret_key'
dynamo_db = AWS::DynamoDB.new(:access_key_id => AWS_ACCESS_KEY, :secret_access_key => AWS_SECRET_KEY)
# create a table (50 read and 50 write capacity units)
# create composite index using hash_key (:user_id) and range_key (:created_at)
@arfon
arfon / create_table_hash_key.rb
Created Jan 21, 2012
AWS DynamoDB table creation (simple primary_key)
View create_table_hash_key.rb
require 'aws-sdk'
AWS_ACCESS_KEY = 'your_aws_access_key'
AWS_SECRET_KEY = 'your_aws_secret_key'
dynamo_db = AWS::DynamoDB.new(:access_key_id => AWS_ACCESS_KEY, :secret_access_key => AWS_SECRET_KEY)
# create a table (50 read and 50 write capacity units)
# create primary key index using hash_key (:id)
@arfon
arfon / gop_simple.rb
Created Jan 21, 2012
Simple Twitterstream example
View gop_simple.rb
require 'tweetstream'
TweetStream.configure do |config|
config.consumer_key = 'consumer_key'
config.consumer_secret = 'consumer_secret'
config.oauth_token = 'oauth_token'
config.oauth_token_secret = 'oauth_token_secret'
config.auth_method = :oauth
config.parser = :json_pure
end
@arfon
arfon / twitterstream_dynamodb.rb
Created Jan 21, 2012
Simple Twitterstream writing to DynamoDB
View twitterstream_dynamodb.rb
require 'aws-sdk'
require 'eventmachine'
require 'tweetstream'
AWS_ACCESS_KEY = 'your_aws_access_key'
AWS_SECRET_KEY = 'your_aws_secret_key'
dynamo_db = AWS::DynamoDB.new(:access_key_id => AWS_ACCESS_KEY, :secret_access_key => AWS_SECRET_KEY)
table = dynamo_db.tables['tweets']
View tweet_counter.rb
require 'aws-sdk'
AWS_ACCESS_KEY = 'your_aws_access_key'
AWS_SECRET_KEY = 'your_aws_secret_key'
dynamo_db = AWS::DynamoDB.new(:access_key_id => AWS_ACCESS_KEY, :secret_access_key => AWS_SECRET_KEY)
table = dynamo_db.tables['tweets']
# don't forget to load the schema
table.load_schema
@arfon
arfon / em_tweetstreamer.rb
Created Jan 21, 2012
Tweet parsing with EventMachine
View em_tweetstreamer.rb
require 'aws-sdk'
require 'eventmachine'
require 'tweetstream'
AWS_ACCESS_KEY = 'your_aws_access_key'
AWS_SECRET_KEY = 'your_aws_secret_key'
dynamo_db = AWS::DynamoDB.new(:access_key_id => AWS_ACCESS_KEY, :secret_access_key => AWS_SECRET_KEY)
table = dynamo_db.tables['tweets']
View sample_queries.rb
require 'aws-sdk'
AWS_ACCESS_KEY = 'your_aws_access_key'
AWS_SECRET_KEY = 'your_aws_secret_key'
dynamo_db = AWS::DynamoDB.new(:access_key_id => AWS_ACCESS_KEY, :secret_access_key => AWS_SECRET_KEY)
table = dynamo_db.tables['tweets']
table.load_schema
View gist:1810357
results = ActiveRecord::Base.connection.execute("select light_curve_id from classifications where zooniverse_user_id=#{user.id}")
results.each { |r| ids << r.first }
ids_to_process = ids.flatten.uniq
require 'enumerator'
unless ids.empty?
ids.each_slice(1000) do |a|
RedisConnect.redis.sadd("zooniverse_user_#{user.id}", *a)
end
end