flat application

1. framework.rb

Merb::Config[:framework] = {
  :application => Merb.root / "application.rb",
  :config => [Merb.root / "config", nil],
  :envrionment => Merb.root / "config/environments",
  :public => [Merb.root / "public", nil],
  :model => Merb.root / "models",
  :view   => Merb.root / "views"
}

2. init.rb

use_orm :datamapper
use_test :rspec
use_template_engine :erb

merb_gems_version = "1.0.7.1"
dm_gems_version   = "0.9.9"

dependency "merb-action-args", merb_gems_version
dependency "merb-assets", merb_gems_version
dependency "merb-helpers", merb_gems_version
dependency "merb-param-protection", merb_gems_version
dependency "merb-exceptions", merb_gems_version

dependency "data_objects", '0.9.10.1'
dependency "dm-aggregates", dm_gems_version
dependency "dm-migrations", dm_gems_version
dependency "dm-timestamps", dm_gems_version
dependency "dm-types", dm_gems_version
dependency "dm-validations", dm_gems_version
dependency "do_mysql", '0.9.10.1'
dependency "dm-is-searchable", dm_gems_version
dependency "dm-sphinx-adapter", '0.6.2'
dependency "dm-core", dm_gems_version


dependency 'merb-cache', merb_gems_version do 
  Merb::Cache.setup do
    unless defined? CACHE_SETUP
      register(:twitter_fragment_store, Merb::Cache::FileStore, :dir => Merb.root / 'cache' / 'fragments')
      register(:twitter_page_store, Merb::Cache::PageStore[Merb::Cache::FileStore], :dir => Merb.root / 'public' / 'page_cache')
      register(:default, Merb::Cache::AdhocStore[:twitter_page_store, :twitter_fragment_store])
        register(:memcached,  Merb::Cache::MemcachedStore, :namespace => "twitter", :servers => ["127.0.0.1:11211"])
   end
   CACHE_SETUP = true
  end
end


gem('twitter4r', '0.3.0')
require 'twitter'
require 'time'
require 'open-uri'
dependency 'nokogiri'
dependency 'shorturl'

Merb::BootLoader.before_app_loads do
   Merb::Slices.config[:fiveruns_tuneup_merb][:api_key] = 'd4dbf40cbeea65asdfasdf4302fcf3f027abe07f3'

end

Merb::BootLoader.after_app_loads do
  DataMapper.setup(:default, :adapter => 'in_memory')
  DataMapper.setup(:search, 'sphinx://localhost:3312')
end

# Move this to application.rb if you want it to be reloadable in dev mode.
Merb::Router.prepare do
  match('/').to(:controller => "twitter_on_merb", :action =>'index')
  match('/about').to(:controller => "twitter_on_merb", :action =>'about')
  match('/commits').to(:controller => "twitter_on_merb", :action =>'commits')
  default_routes
end

# I copied environments folder from normal app into config folder
Merb::Config.use { |c|

  c[:use_mutex]           = false,
  c[:session_store]       = 'cookie',
  c[:session_id_key]      = '_twitter_on_merb_session_id',
  c[:session_secret_key]  = '6f61c1a0968b761a4535f82d9da4619dc2ad461e'

}

3. deploy.rb

set :application, "merb.kicks-ass.org"
set :domain, "**.***.***.**"
set :user, 'deploy'
set :use_sudo, false
set :scm, :git
set :runner, user
set :ssh_options, { :forward_agent => true }
default_run_options[:pty] = true
set :port, ****** #ssh port
set :repository,  "ssh://deploy@**.***.***.**:******/git/twitter.git"
set :deploy_to, "/home/deploy/repos/#{application}"
set :deploy_via, :remote_cache
set :copy_exclude, ['.git', 'Capfile', 'config/deploy.rb']
# default
role :app, application, :cron => true
role :web, application
role :db, application, :primary => true

# server restart task
namespace :deploy do
  desc "Restart Application"
  task :restart, :roles => :app do
    run "touch #{current_path}/tmp/restart.txt"
  end
end

namespace :deploy do
  desc "Copy db conf"
  task :copy_database_configureation, :roles => :app  do
    production_db_config = '#{shared_path}/database.yml'
    run "cp #{production_db_config} #{release_path}/config/database.yml"
  end
end

# cron job. to scrape and save periodically.
namespace :cron do
  desc "cron job start"
  task :start, :roles => :app  do
    cron_tab = "#{shared_path}/cron.tab"
    run "mkdir -p #{shared_path}/log/cron"
    
    require 'erb'
    template = File.read("config/cron.erb")
    file = ERB.new(template).result(binding)
    put file, cron_tab, :mode => 0644
    
    # merge with the current crontab
    # fails with an empty crontab, which is acceptable
    run "crontab -l >> #{cron_tab}" rescue nil
    
    # install the new crontab
    run "crontab #{cron_tab}"
  end
end


namespace :cron do
  desc "cron job stop"
  task :stop, :roles => :app do
    cron_tmp = "#{shared_path}/cron.old"
    cron_tab = "#{shared_path}/cron.tab"   
    begin
      # dump the current cron entries
      run "crontab -l > #{cron_tmp}"
      
      # remove any lines that contain the application name
      run "awk '{if ($0 !~ /#{application}/) print $0}' " +
        "#{cron_tmp} > #{cron_tab}"
        
      # replace the cron entries
      run "crontab #{cron_tab}"
    rescue
      # fails with an empty crontab, which is acceptable
    end
    
    # clean up
    run "rm -rf #{cron_tmp}"
  end
end

#before "deploy", "deploy:copy_database_configureation"

after "deploy:stop", "cron:stop"
before "deploy", "cron:stop"
after "deploy", "cron:start"

periodic_joc.rake

namespace :chores do
    desc "Scrapes on every 30 min"
    task :evenry30minutes =>  :merb_env do
    chore("Every 30 minutes") do
      p Tweet.scrape
    end
  end

  desc "Scrapes on every hour"
  task :hourly =>  :merb_env   do
    chore("Hourly") do
      # Your Code Here
    end
  end
  desc "Scrapes on every day"
  task :daily  =>  :merb_env do
    chore("Daily") do
      # Your Code Here
    end
  end
  desc "Scrapes on every week"
  task :weekly  =>  :merb_env do
    chore("Weekly") do
      # Your Code Here
    end
  end
  
  def chore(name)
    puts "#{name} Task Invoked: #{Time.now}"
    yield
    puts "#{name} Task Finished: #{Time.now}"
  end

end

====================================================
config/cron.erb

*/10 * * * * MERB_ENV='production' rake -f <%= current_path %>/Rakefile chores:evenry30minutes  >> <%= deploy_to %>/shared/log/cron/evenry30minutes.txt

00 * * * * MERB_ENV='production' rake -f <%= current_path %>/Rakefile chores:hourly >> <%= deploy_to %>/shared/log/cron/hourly.txt

01 0 * * * MERB_ENV='production' rake -f <%= current_path %>/Rakefile chores:daily >> <%= deploy_to %>/shared/log/cron/daily.txt

02 0 * * 1 MERB_ENV='production' rake -f <%= current_path %>/Rakefile chores:weekly >> <%= deploy_to %>/shared/log/cron/weekly.txt

tweet.rb

# referenced API.
# http://merbapi.com/classes/Merb/BootLoader/BuildFramework.html
class Tweet
  include DataMapper::SphinxResource

  property :id, Serial
  property :message, Text, :length => 255, :nullable => false
  property :name, String, :length => 20
  property :category, String, :length => 20
  property :link, String
  property :date, String
  property :created_at,  DateTime
  property :updated_on, DateTime
 
  is :searchable
    repository(:search) do
      index :tweets
      index :tweets_delta, :delta => true

      # Sphinx attributes to sort include/exclude by.
      attribute :updated_at, DateTime
  end


  def self.recent(start_at = 1.week.ago)
    all(:created_at.gte => start_at, :order => [:date.desc])
  end

  def self.latest
    first(:order => [:created_at.desc])
  end  



  def self.scrape
    @edgemerb_commits = []
    merb = 'http://github.com/wycats/merb/commits/1.0.x'
    @negotiation_commits = []
    negotiation = 'http://github.com/wycats/rails/commits/content_negotiation'
    @katz_blog = []
    blog= 'http://yehudakatz.com'
    self.get_commits_and_bark(merb, @edgemerb_commits, "edgemerb")
    self.get_commits_and_bark(negotiation, @negotiation_commits, "negotiation")
    self.get_blog_and_bark(blog, @katz_blog)
  end    
    
    

#parsing github commits 
  def self.get_commits_and_bark(url, array, category)
    self.connect
      @doc  = Nokogiri::HTML(open(url))
      @doc.css('div.human').each do |h|
         h.search('div.message').each {|k| @message = k.content}
         h.search('div.name').each do |k| 
           @name =  (k.content).to_s.strip.gsub(" (author)", "").gsub("(committer)","")
         end
         h.search('div.date').each do |k| 
           @date = k.content
           @new_date = Time.parse(@date.strip)
         end
         h.search('pre a[href]').each do |k| 
           @link =  (k.attributes)
           begin
             @new_link =  ShortURL.shorten("http://github.com" + @link.to_s.strip.gsub("href", ''), :rubyurl)
           rescue  => e
             @new_link =  ShortURL.shorten("http://github.com" + @link.to_s.strip.gsub("href", ''), :tinyurl)
           rescue Timeout::Error => e
              @new_link =  "http://github.com" + @link.to_s.strip.gsub("href", '')
           end
         end
         array <<{:message => @message, :name => @name, :date => @new_date, :link => @new_link, :category => category}           
      end  
      array.slice!(4, array.length) 
      self.initialize
      array.each do |tweet|
        if not self.search(:message => tweet[:message]) then
          create(tweet)
        end
      end
  end


  
#parsing a blog  
  def self.get_blog_and_bark(url, array)

      @doc  = Nokogiri::HTML(open(url))
      @doc.css('div.entry').each do |h|
        @name = "Katz"
        h.search('h3.entrytitle').each {|k| @title = k.content }        
        h.search('h3 a[href]').each do |k| 
          @link =  (k.attributes)
           begin
          @new_link =  ShortURL.shorten(@link.to_s.strip.gsub("href", '').gsub('relbookmark',''), :rubyurl)
           rescue  => e
          @new_link =  ShortURL.shorten(@link.to_s.strip.gsub("href", '').gsub('relbookmark',''), :tinyurl)
           rescue Timeout::Error => e
              @new_link = @link.to_s.strip.gsub("href", '').gsub('relbookmark','')
           end          
        end
        h.search('div.meta').each do |k| 
          @date = k.content
          @new_date = Time.parse(@date.strip)
        end
        array <<{:message => @title, :name => @name, :date=> Time.now, :category => "Katz's blog", :link => @new_link}
      end
      array.slice!(10, array.length)
      array.each do |tweet|
        self.create(tweet)
      end

  end
  

#create tweets
#Since twitter API does not provide unique message check,
#I had to build db to validiate it.
  def self.create(tweet)  
    @tweet = self.new(tweet)
    if @tweet.save

        begin
         self.post(tweet)
        rescue Twitter::RESTError => re
          unless re.code == "403"
            puts re
          end
        end #begin-rescue end 

    else
      puts "failed to save them"
    end    
    return @tweet
  end
  
  def self.post(tweet)
    @poster = String.new
    sleep(15) # sleep. because of twitter api limit
    @poster = '[' + tweet[:category] + ']' + tweet[:message] +'('+ tweet[:name]  +')'+ tweet[:link]
    status =@post_client.status(:post, @poster)
  end

  def self.connect
    @post_client = Twitter::Client.new(:login => 'edgemerb', :password => 'merbonrails')
  end
end