This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import mwlib.parser.nodes | |
import mwlib.uparser | |
import codecs | |
with codecs.open('us.dat', 'rb', encoding='utf-8') as fp: | |
text = fp.read() | |
article = mwlib.uparser.parseString(title='us', raw=text) | |
nodes = article.find(mwlib.parser.nodes.ArticleLink) | |
for n in nodes: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
5 procs started on release0.13.3 @ 13:45 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var downstream = function(buzz_id, root_id, limit) { | |
if (root_id == null){ | |
return db.user_edge_v2.find({ | |
"buzzid": buzz_id | |
}).count() | |
} | |
var limit = typeof limit != 'undefined' ? 1000 : limit; | |
var seen = {root_id: true}; | |
var nodeq = [root_id]; | |
var downstream_count = 0; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def should_add(self, edge): | |
return all([ | |
self.would_be_leaf(edge), | |
self.is_earliest_parent(edge)]) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
conn = StrictRedis() | |
def online_size(sender, tree_name=None, edge=None): | |
conn.incr('{0}.size'.format(tree_name)) | |
def online_depth(sender, tree_name=None, edge=None): | |
with conn.lock('{0}.depth'.format(tree_name)): | |
current_depth = conn.get('{0}.depth'.format(tree_name)) or 0 | |
if int(current_depth) < edge.generation: | |
conn.set('{0}.depth'.format(tree_name), edge.generation) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class Edge(object): | |
def __init__(self, parent, child, created_at): | |
self.parent = parent | |
self.child = child | |
self.created_at = created_at | |
self.generation = 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def get_descendants(node): | |
unexplored = deque([node]) | |
while unexplored: | |
node = unexplored.popleft() | |
yield node | |
children = node.children | |
for child in children: | |
unexplored.append(child) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
graph = [] | |
for name in xrange(6): | |
graph.append(Node(name)) | |
graph[0].add_child(graph[1]) | |
graph[0].add_child(graph[2]) | |
graph[2].add_child(graph[3]) | |
graph[2].add_child(graph[4]) | |
graph[4].add_child(graph[5]) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class Node(object): | |
def __init__(self, name): | |
self.name = name | |
self.children = [] | |
def add_child(self, child): | |
self.children.append(child) | |
def __repr__(self): | |
return '<{0}>'.format(self.name) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def get_descendants(node, A): | |
unexplored = deque([node]) | |
while unexplored: | |
node = unexplored.popleft() | |
yield node | |
children = A.getrow(node).indices | |
for child in children: | |
unexplored.append(child) |