Automated Scraping with Heroku and Sinatra

Automated Scraping with Heroku and Sinatra

mkdir wpgskeeter
cd wpgskeeter
git init
heroku create wpgskeeter
git add .
git commit -m 'Initial commit'

Create a .gems file:
sinatra
nokogiri
compass
rdiscount
activerecord
activerecord-postgres-adapter


Create a config.ru file:
require 'sinatra_main'
run Sinatra::Application

Command line:
heroku console
ENV['database_url']

Create a config/database.yml file:
production: 
    encoding: unicode
    adapter: postgresql
    port: 5432
    host: 'host'
    database: 'db'
    username: 'user'
    password: 'pass'

Create a sinatra_main.rb file:
require 'sinatra'
require 'nokogiri'
require 'sass'
require 'haml'
require 'compass'

ActiveRecord::Base.establish_connection(YAML::load(File.open('config/database.yml'))['production'])

get '/' do
  "Hello World!"
end

Command line:
heroku addons:add cron:daily
heroku addons:add sendgrid:free

Create your Rakefile:
task :environment do
  require 'active_record'
  require 'pony'
  ActiveRecord::Base.establish_connection(YAML::load(File.open('config/database.yml'))['production'])
end

namespace :db do
  desc "Migrate the database"
  task(:migrate => :environment) do
    ActiveRecord::Base.logger = Logger.new(STDOUT)
    ActiveRecord::Migration.verbose = true
    ActiveRecord::Migrator.migrate("db/migrate")
  end
end

task :cron => :environment do
    Pony.mail(:to => 'stungeye@gmail.com',
            :from => 'me@example.com',
            :subject => 'hi',
            :body => 'Hello there.',
            :via => :smtp, :via_options => {
              :address        => 'smtp.sendgrid.net',
              :port           => '25',
              :authentication => :plain, 
              :user_name      => ENV['SENDGRID_USERNAME'],
              :password       => ENV['SENDGRID_PASSWORD'],
              :domain         => ENV['SENDGRID_DOMAIN']}
  )
end


Test out the cron manually from the command-line:
heroku rake cron