wlmcewen

# Using set here doesn't work, as the git tasks are loaded later.
#   set :git_environmental_variables, ->() {
#   ...


task :override_defaults
  set :git_environmental_variables, ->() {
    {
      git_askpass: "/bin/echo",
      git_ssh:     "/home/deploy/util/gitkey/wrapper"

wlmcewen

UPDATE "subscriber_source" ss 
SET    subscriber_source_approved = 't' 
WHERE  subscriber_source_approved = 'f' 
       AND subscriber_id in ('SOME IDS...')
       AND source_id = '20150707140016042989000000' 
       AND 0 < (SELECT Count(*) 
                FROM   newsfeed, newsfeed_item
                WHERE  newsfeed.newsfeed_id = newsfeed_item.newsfeed_id 
                       AND ss.subscriber_id = newsfeed.subscriber_id 
                       AND newsfeed_item.source_id = 

wlmcewen

# maybe we can handle the cascade db side like so. 

select idx(array[22740185, 19161654], notification_id), * from notification where notification_id in (19161654, 52740185) order by idx(array[52740185, 19161654], notification_id) nulls last;

# perhaps we cascade by throwing the contact, agent and mls in a subselect on this collection ordered smartly

prospector_alpha=# select subscriber_id, master_owner_id, r.mls_id from resource r inner join newsfeed f on r.resource_id = f.resource_id where r.resource_id = '20151215223132896052000000' and f.subscription_name in (select subscription_name from newsfeed f2 where subscriber_id in (f.subscriber_id,r.master_owner_id, f.mls_id) and subscription_name = 'Subscription kaplan search ash' limit 1);
       subscriber_id        |      master_owner_id       |           mls_id           
----------------------------+----------------------------+----------------------------
 20150320225609690019000000 | 20150320225609690019000000 | 20141203195816310156000000

wlmcewen

--- QUERY1 : Prospector newsfeed scheduled email filter

SELECT source_id, newsfeed_item_id FROM "newsfeed_item" WHERE "newsfeed_item"."newsfeed_id" = 9390420 AND (0 < (select count(*) from newsfeed_item_event e where e.newsfeed_item_id = newsfeed_item.newsfeed_item_id and event in (1,7,4,2,0,5) and e.event_created_at >= '2015-12-15 19:39:34.940333')) ORDER BY (select event_created_at from newsfeed_item_event where newsfeed_item.newsfeed_item_id = newsfeed_item_event.newsfeed_item_id order by event_created_at desc limit 1) desc nulls last, newsfeed_item.last_event_at desc nulls last, newsfeed_item.newsfeed_item_id desc LIMIT 50;


--- QUERY2 : Ordering... subscribers? 
SELECT subscriber_id, max ( GREATEST ( last_event_at, event_created_at ) ) AS last_event_at FROM "newsfeed_item" LEFT OUTER JOIN "newsfeed_item_event" ON "newsfeed_item_event"."newsfeed_item_id" = "newsfeed_item"."newsfeed_item_id" WHERE "newsfeed_item"."source_type" = '' AND "newsfeed_item"."subscriber_id" IN ( ... ) AND "newsfeed_item"."notif

wlmcewen

--- old
SELECT  newsfeed_item_id FROM "newsfeed_item"  WHERE (newsfeed_item_id IN (select distinct on (source_id) newsfeed_item_id from newsfeed_item  where newsfeed_item.subscriber_id='20150623231104072663000000' AND ("notification_sent" = 't' AND "newsfeed_item_approved" = 't') order by source_id, (select event_created_at from newsfeed_item_event where newsfeed_item.newsfeed_item_id = newsfeed_item_event.newsfeed_item_id order by event_created_at desc limit 1) desc nulls last, last_event_at desc nulls last)) ORDER BY (select event_created_at from newsfeed_item_event where newsfeed_item.newsfeed_item_id = newsfeed_item_event.newsfeed_item_id order by event_created_at desc limit 1) desc nulls last, newsfeed_item.last_event_at desc nulls last, newsfeed_item.newsfeed_item_id desc LIMIT 200 OFFSET 0

---new
SELECT  newsfeed_item.source_id, MIN(newsfeed_item.newsfeed_item_id) as newsfeed_item_id, MAX(COALESCE(event_created_at, newsfeed_item_created_at)) as last_event_at FROM "newsfeed_item" LEFT OUTER JOIN newsfe

wlmcewen

# Note the uniq pid files at the end

/usr/bin/env bundle exec sidekiqctl quiet /home/httpd/prospector/prod/current/tmp/sidekiq.pid-match-3
/usr/bin/env bundle exec sidekiqctl quiet /home/httpd/prospector/prod/current/tmp/sidekiq.pid-match-4
/usr/bin/env bundle exec sidekiqctl quiet /home/httpd/prospector/prod/current/tmp/sidekiq.pid-match-5

/usr/bin/env bundle exec sidekiq -d -i 0 -e production -C /home/httpd/prospector/prod/current/config/sidekiq.yml -L /home/httpd/prospector/prod/current/log/sidekiq.log -g prospector-prod -c 10 -q newsfeeds,4 -q processing,1 -P /home/httpd/prospector/prod/current/tmp/sidekiq.pid-0
/usr/bin/env bundle exec sidekiq -d -i 0 -e production -C /home/httpd/prospector/prod/current/config/sidekiq.yml -L /home/httpd/prospector/prod/current/log/sidekiq.log -g prospector-prod -c 10 -q newsfeeds,4 -q processing,1 -P /home/httpd/prospector/prod/current/tmp/sidekiq.pid-1
/usr/bin/env bundle exec sidekiq -d -i 0 -e production -C /home/httpd/prospector/prod/current/config/sidekiq.yml -L /

wlmcewen

SELECT  
    "chs_coresearch"."listingkey",
    "chs_coresearch"."modificationtimestamp",
    "chs_coresearch"."sparkmodificationtimestamp" 
FROM "chs_coresearch"  
WHERE 
(
    "chs_coresearch"."listingkey" in 
    (
        select distinct "chs_coresearch"."listingkey" from chs_coresearch 

wlmcewen

require 'uri'
require 'net/http'
require 'net_json'

class N
  include NetJson

end

uri = URI.parse("http://internal.flexmls.com/flexiapi/prefs/auto_inbox_exstatus")

wlmcewen

My largest Sidekiq application had a memory leak and I was able to find and fix it in just few hours spent on analyzing Ruby's heap. In this post I'll show my profiling setup.

As you might know Ruby 2.1 introduced a few great changes to ObjectSpace, so now it's much easier to find a line of code that is allocating too many objects. Here is great post explaining how it's working.

I was too lazy to set up some seeding and run it locally, so I checked that test suite passes when profiling is enabled and pushed debugging to production. Production environment also suited me better since my jobs data can't be fully random generated.

So, in order to profile your worker, add this to your Sidekiq configuration:

if ENV["PROFILE"]

wlmcewen

SparkApi:048:0*   post("/savedsearches", "Name" => "Wade's bagley test", "Filter" => "City Eq 'Bagley' And MlsStatus Eq 'Active' And PropertyType Eq 'A'")
[{"ResourceUri"=>"/v1/savedsearches/20160930110443923409000000", "Id"=>"20160930110443923409000000", "OwnerId"=>"20081211210137031788000000", "Name"=>"Wade's bagley test", "Description"=>nil, "ContactIds"=>[], "UnsubscribedContactIds"=>[], "ModificationTimestamp"=>"2016-09-30T11:04:43-05:00", "Filter"=>"City Eq 'Bagley' And MlsStatus Eq 'Active' And PropertyType Eq 'A'", "QuickSearchId"=>nil, "Tags"=>[], "Provided"=>false}]
SparkApi:049:0> get("/newsfeeds/20151117150944963050973798")[{"Id"=>"20151117150944963050973798", "ResourceUri"=>"/v1/newsfeeds/20151117150944963050973798", "OwnerId"=>"20081211210137031788000000", "ModificationTimestamp"=>"2016-04-13T18:15:52Z", "CreatedTimestamp"=>"2015-11-17T21:09:44Z", "Events"=>[nil], "UseDefaultEvents"=>false, "Type"=>"SavedSearch", "Name"=>"alpha agent scheduled OpenHouse", "Active"=>true, "NotificationsActive"=>