Skip to content

Instantly share code, notes, and snippets.

Created December 27, 2012 01:55
Show Gist options
  • Select an option

  • Save anonymous/4384778 to your computer and use it in GitHub Desktop.

Select an option

Save anonymous/4384778 to your computer and use it in GitHub Desktop.
a Neo4j::Rails::Model This is exhibiting a memory leak in the #import_connections method (which wraps #process_connections).
require 'linkedin'
class Person < Neo4j::Rails::Model
include LinkedIn
FIELDS = 'id,first-name,last-name,headline,location,positions,educations,picture-url,site-public-profile-request,industry,email-address'
has_n(:friends)
property :firstname
property :lastname
property :headline
property :industry
property :company
property :raw_profile
property :oauth_token
property :oauth_secret
property :updated_at
property :created_at
property :imported_connections_at, :type => DateTime
property :imported_edges_at, :type => DateTime
property :linkedin_id, :index => :exact, :unique => true
validates :linkedin_id, :exclusion => { :in => %w(private) }
validates :firstname, :presence => true, :allow_blank => false
validates :lastname, :presence => true, :allow_blank => false
before_validation :update_properties
def self.contributors
all.select { |person| person.is_contributor? }
end
def self.updateable
all.select { |person| person.is_contributor? && person.needs_refresh? }
end
def self.edges_updateable
all.select { |person| person.is_contributor? && person.imported_connections_at } # && person.needs_edges_refresh? }
end
def name
"#{self.firstname} #{self.lastname}"
end
def update_properties
self.raw_profile ||= get("/v1/people/~:(#{FIELDS})")
self.errors[:base] << "Invalid profile" if profile['status'] == 401
self.firstname = profile['firstName']
self.lastname = profile['lastName']
self.headline = profile['headline']
self.industry = profile['industry']
self.linkedin_id = profile['id']
self.oauth_token ||= profile['token']
self.oauth_secret ||= profile['secret']
end
def profile
@profile ||= JSON.parse(raw_profile)
end
def import_connections
url = "/v1/people/~/connections:(#{FIELDS})"
url += "?modified=new&modified-since=#{self.imported_connections_at.to_i*1000}" if self.imported_connections_at
prev_count = self.friends.count
connections_object = JSON.parse(get(url))
processed_count = process_connections(connections_object)
"#{self.name} [#{prev_count}+#{processed_count}=#{self.friends.count}]"
end
def import_edges
edgecount = 0
rels_needing_edges = self.friends_rels.to_a.select { |r| r[:imported_edges_at] == nil }
id_list = rels_needing_edges.map { |r| r.end_node.linkedin_id }
return unless id_list && id_list.any?
puts "#{self.name}: need #{id_list.count} out of #{self.friends.count}"
#id_list = self.friends.map { |f| f.linkedin_id }
while (id_list.any? && (l = id_list.shift(500)) ) do
ids = URI.escape(l.join(','))
edges_object = JSON.parse(get("/v1/people::(#{ids}):(id,relation-to-viewer)"))
edgecount += process_edges(edges_object)
end
self.imported_edges_at = Time.now.utc
self.save
"#{self.name} [#{self.friends.count} friends, imported #{edgecount} edges]"
end
def is_contributor?
oauth_token && oauth_secret
end
def needs_refresh?
self.imported_connections_at.nil? || (self.imported_connections_at < 3.hours.ago)
end
def needs_edges_refresh?
self.imported_edges_at.nil? || (self.imported_edges_at < 24.hours.ago)
end
# Try to ensure that friendships are unique
def find_or_create_friendship(p2)
# puts "finding rel #{self.name} <-> #{p2.name}"
existing = self.friends_rels.to_other(p2)
if existing.any?
existing.first
else
if p2 && (!p2.name.blank?)
Connection.create_symmetric(self,p2)
end
end
end
private
def process_connections(o)
(puts "#{self.name}: #{o['message']}" and return 0) if o['status'] && o['status'].to_s[/^4/]
list = o['values']
count = 0
if list && list.any?
puts "#{self.name} [#{o['_total']}]"
Person.transaction do |tx|
list.each do |p|
next if p['id']=='private' || p['firstName'].blank? || p['lastName'].blank?
friend = Person.find_by_linkedin_id(p['id'])
friend ||= Person.create(:raw_profile => p.to_json)
self.find_or_create_friendship(friend)
count += 1
end
end
Neo4j::Rails::Model.close_lucene_connections
end
self.imported_connections_at = Time.now.utc
self.save
count
end
def process_edges(o)
(puts "#{self.name}: #{o['message']}" and return 0) if o['status'] && o['status'].to_s[/^4/]
return 0 unless edges = o['values']
count = 0
edges.each do |e|
source = Person.find_by_linkedin_id(e['id'])
next unless e['relationToViewer'] && (connections = e['relationToViewer']['connections']['values'])
connections.each do |c|
dest_id = c['person']['id']
next if dest_id=='private'
Person.transaction do |tx|
dest = Person.find_by_linkedin_id(dest_id)
source.find_or_create_friendship(dest)
end
end
Person.transaction do |tx|
source_friendship = self.friends_rels.to_other(source).first
source_friendship[:imported_edges_at] = Time.now.utc.to_i
end
count += source.friends.count
source.save
end
puts "#{self.name}: #{count} edges from #{o['_total']} connections"
count
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment