seamusabshere

#!/usr/bin/env ruby

PRESUMPTIVE_RAILS_ROOT = '/data/data1/current'
Dir.chdir PRESUMPTIVE_RAILS_ROOT

require 'rubygems' # do this after changing dirs or else bundler won't be able to find .bundle
gem 'seamusabshere-daemon-spawn', '0.2.6' # otherwise we don't get timeout kill -9
require 'daemon_spawn'

PID_PATH = '/data/data1/shared/pids/delayed_job.pid'

seamusabshere

# http://jetpackweb.com/blog/2010/05/19/making-monit-delayed_job-and-bundler-play-nice-together/
check process my_app_delayed_job_1
  with pidfile /data/my_app/shared/pids/delayed_job.pid
  start program = "/usr/bin/env HOME=/home/deploy PATH=/usr/bin:$PATH RAILS_ENV=production /etc/init.d/delayed_job.sh start" as uid deploy and gid deploy
  stop program =  "/usr/bin/env HOME=/home/deploy PATH=/usr/bin:$PATH RAILS_ENV=production /etc/init.d/delayed_job.sh stop" as uid deploy and gid deploy with timeout 90 seconds
  if totalmem is greater than 256 MB for 10 cycles then restart
  group my_app_delayed_job

seamusabshere

#!/usr/bin/env ruby

PRESUMPTIVE_RAILS_ROOT = '/data/my_app/current'
Dir.chdir PRESUMPTIVE_RAILS_ROOT

require "rubygems"
require "bundler/setup"
require 'daemon_spawn' # use 0.4.0

PID_PATH = '/data/my_app/shared/pids/delayed_job.pid'

seamusabshere

require 'test_helper'

class Shop  < ActiveRecord::Base
end

class ItemDetail < ActiveRecord::Base
  set_primary_key :row_hash
    
    data_miner do
    import 'A csv file with no identifier', :url => 'http://imports.s3.amazonaws.com/products_no_id.csv', :errata => 'http://spreadsheets.google.com/pub?key=tF58sAgqkY214EFBMdYCIUw', :delimiter => ';' do

seamusabshere

diff --git a/data_miner_test.rb b/data_miner_test.rb
index 69c9cd5..fca902c 100644
--- a/data_miner_test.rb
+++ b/data_miner_test.rb
@@ -4,12 +4,12 @@ class Shop  < ActiveRecord::Base
 end
 
 class ItemDetail < ActiveRecord::Base
-  set_primary_key :id
+  set_primary_key :row_hash

seamusabshere

#!/usr/bin/env ruby

PRESUMPTIVE_RAILS_ROOT = "/data/[APP_NAME]/current"
LOG_PATH = "/data/[APP_NAME]/shared/log/delayed_job_singleton.log"

Dir.chdir PRESUMPTIVE_RAILS_ROOT

require 'home_run'
require "rubygems"
require "bundler/setup"

seamusabshere

vidalia:~/cm1 (statistics) $ rails console
Loading development environment (Rails 3.0.6)
ruby-1.9.2-p180 :001 > m = AutomobileTrip.method(:run_data_miner_on_belongs_to_associations)
 => #<Method: AutomobileTrip([...])(ActiveRecord::Base).run_data_miner_on_belongs_to_associations> 
ruby-1.9.2-p180 :002 > m.source_location 
 => ["/Users/seamus/.rvm/gems/ruby-1.9.2-p180/bundler/gems/earth-3b880b7687ac/lib/earth/active_record_ext.rb", 3] 

seamusabshere

#!/usr/bin/env ruby

require 'rubygems'
require 'remote_table'

# include to_regexp stuff from https://gist.github.com/926425

def manufacturer_whitelist?(candidate)
  @manufacturer_whitelist ||= RemoteTable.new(:url => AAAAAAA)
  @manufacturer_whitelist.any? { |record| record['Manufacturer'].to_regexp.match(candidate) }

seamusabshere

#
# DEPRECATED - and probably broken
# use to_regexp gem instead
# https://github.com/seamusabshere/to_regexp
# https://rubygems.org/gems/to_regexp
#

class String
  REGEXP_DELIMITERS = {
    '%r{' => '}',

seamusabshere

# parallel 2

## plain

gargantuan1:~ $ http_load -parallel 2 -fetches 3000 urls.txt 
3000 fetches, 2 max parallel, 54000 bytes, in 28.6926 seconds
18 mean bytes/connection
104.557 fetches/sec, 1882.02 bytes/sec
msecs/connect: 0.296469 mean, 9.802 max, 0.04 min
msecs/first-response: 18.8286 mean, 27.964 max, 9.277 min