Created
December 27, 2012 01:55
-
-
Save anonymous/4384778 to your computer and use it in GitHub Desktop.
a Neo4j::Rails::Model This is exhibiting a memory leak in the #import_connections method (which wraps #process_connections).
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| require 'linkedin' | |
| class Person < Neo4j::Rails::Model | |
| include LinkedIn | |
| FIELDS = 'id,first-name,last-name,headline,location,positions,educations,picture-url,site-public-profile-request,industry,email-address' | |
| has_n(:friends) | |
| property :firstname | |
| property :lastname | |
| property :headline | |
| property :industry | |
| property :company | |
| property :raw_profile | |
| property :oauth_token | |
| property :oauth_secret | |
| property :updated_at | |
| property :created_at | |
| property :imported_connections_at, :type => DateTime | |
| property :imported_edges_at, :type => DateTime | |
| property :linkedin_id, :index => :exact, :unique => true | |
| validates :linkedin_id, :exclusion => { :in => %w(private) } | |
| validates :firstname, :presence => true, :allow_blank => false | |
| validates :lastname, :presence => true, :allow_blank => false | |
| before_validation :update_properties | |
| def self.contributors | |
| all.select { |person| person.is_contributor? } | |
| end | |
| def self.updateable | |
| all.select { |person| person.is_contributor? && person.needs_refresh? } | |
| end | |
| def self.edges_updateable | |
| all.select { |person| person.is_contributor? && person.imported_connections_at } # && person.needs_edges_refresh? } | |
| end | |
| def name | |
| "#{self.firstname} #{self.lastname}" | |
| end | |
| def update_properties | |
| self.raw_profile ||= get("/v1/people/~:(#{FIELDS})") | |
| self.errors[:base] << "Invalid profile" if profile['status'] == 401 | |
| self.firstname = profile['firstName'] | |
| self.lastname = profile['lastName'] | |
| self.headline = profile['headline'] | |
| self.industry = profile['industry'] | |
| self.linkedin_id = profile['id'] | |
| self.oauth_token ||= profile['token'] | |
| self.oauth_secret ||= profile['secret'] | |
| end | |
| def profile | |
| @profile ||= JSON.parse(raw_profile) | |
| end | |
| def import_connections | |
| url = "/v1/people/~/connections:(#{FIELDS})" | |
| url += "?modified=new&modified-since=#{self.imported_connections_at.to_i*1000}" if self.imported_connections_at | |
| prev_count = self.friends.count | |
| connections_object = JSON.parse(get(url)) | |
| processed_count = process_connections(connections_object) | |
| "#{self.name} [#{prev_count}+#{processed_count}=#{self.friends.count}]" | |
| end | |
| def import_edges | |
| edgecount = 0 | |
| rels_needing_edges = self.friends_rels.to_a.select { |r| r[:imported_edges_at] == nil } | |
| id_list = rels_needing_edges.map { |r| r.end_node.linkedin_id } | |
| return unless id_list && id_list.any? | |
| puts "#{self.name}: need #{id_list.count} out of #{self.friends.count}" | |
| #id_list = self.friends.map { |f| f.linkedin_id } | |
| while (id_list.any? && (l = id_list.shift(500)) ) do | |
| ids = URI.escape(l.join(',')) | |
| edges_object = JSON.parse(get("/v1/people::(#{ids}):(id,relation-to-viewer)")) | |
| edgecount += process_edges(edges_object) | |
| end | |
| self.imported_edges_at = Time.now.utc | |
| self.save | |
| "#{self.name} [#{self.friends.count} friends, imported #{edgecount} edges]" | |
| end | |
| def is_contributor? | |
| oauth_token && oauth_secret | |
| end | |
| def needs_refresh? | |
| self.imported_connections_at.nil? || (self.imported_connections_at < 3.hours.ago) | |
| end | |
| def needs_edges_refresh? | |
| self.imported_edges_at.nil? || (self.imported_edges_at < 24.hours.ago) | |
| end | |
| # Try to ensure that friendships are unique | |
| def find_or_create_friendship(p2) | |
| # puts "finding rel #{self.name} <-> #{p2.name}" | |
| existing = self.friends_rels.to_other(p2) | |
| if existing.any? | |
| existing.first | |
| else | |
| if p2 && (!p2.name.blank?) | |
| Connection.create_symmetric(self,p2) | |
| end | |
| end | |
| end | |
| private | |
| def process_connections(o) | |
| (puts "#{self.name}: #{o['message']}" and return 0) if o['status'] && o['status'].to_s[/^4/] | |
| list = o['values'] | |
| count = 0 | |
| if list && list.any? | |
| puts "#{self.name} [#{o['_total']}]" | |
| Person.transaction do |tx| | |
| list.each do |p| | |
| next if p['id']=='private' || p['firstName'].blank? || p['lastName'].blank? | |
| friend = Person.find_by_linkedin_id(p['id']) | |
| friend ||= Person.create(:raw_profile => p.to_json) | |
| self.find_or_create_friendship(friend) | |
| count += 1 | |
| end | |
| end | |
| Neo4j::Rails::Model.close_lucene_connections | |
| end | |
| self.imported_connections_at = Time.now.utc | |
| self.save | |
| count | |
| end | |
| def process_edges(o) | |
| (puts "#{self.name}: #{o['message']}" and return 0) if o['status'] && o['status'].to_s[/^4/] | |
| return 0 unless edges = o['values'] | |
| count = 0 | |
| edges.each do |e| | |
| source = Person.find_by_linkedin_id(e['id']) | |
| next unless e['relationToViewer'] && (connections = e['relationToViewer']['connections']['values']) | |
| connections.each do |c| | |
| dest_id = c['person']['id'] | |
| next if dest_id=='private' | |
| Person.transaction do |tx| | |
| dest = Person.find_by_linkedin_id(dest_id) | |
| source.find_or_create_friendship(dest) | |
| end | |
| end | |
| Person.transaction do |tx| | |
| source_friendship = self.friends_rels.to_other(source).first | |
| source_friendship[:imported_edges_at] = Time.now.utc.to_i | |
| end | |
| count += source.friends.count | |
| source.save | |
| end | |
| puts "#{self.name}: #{count} edges from #{o['_total']} connections" | |
| count | |
| end | |
| end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment