Skip to content

Instantly share code, notes, and snippets.

@treystout
Created August 2, 2012 21:28
Show Gist options
  • Save treystout/3240817 to your computer and use it in GitHub Desktop.
Save treystout/3240817 to your computer and use it in GitHub Desktop.
General purpose zset union/intersect code (probably too meta)
#!/usr/bin/env python
import lib.akn.config as cfg
cfg.init(False)
from lib.akn.exceptions import CallError
def __zset_fetch(bind_pairs, start=0, end=29, count=False,
reverse=True, ids_only=False, operator="union", **kwargs):
"""General purpose tool for doing zset traversal in redis without abusing
local memory too much. To call it, you send in pairs consisting of
* a string representing a zset's key
* an optional variable to interpolate into the first element of the pair
If there is more than 1 pair, then the operator kwarg comes into play.
Currently, union and intersection are supported via the native ZUNIONSTORE and
ZINTERSTORE redis commands, respectively.
"""
post_ids = []
keys = []
print "zset featch on pairs:", bind_pairs
for pair in bind_pairs:
if len(pair) == 1:
by_what = pair[0]
elif len(pair) == 2:
by_what, some_id = pair
else:
raise CallError('each element of bind_pairs must be a 1 or 2 tuple')
if by_what == "featured":
keys.append("featured:by_date_created")
elif by_what == "usage_type":
keys.append("usage_type:%s:by_date_created" % some_id)
elif by_what == "media_type":
keys.append("media_type:%s:by_date_created" % some_id)
elif by_what == "spotlight":
keys.append("user:%s:spotlight_posts" % some_id)
elif by_what == 'hashtag':
keys.append("hashtag:%s:posts" % some_id)
else:
raise CallError('cannot get posts by %s' % by_what)
with cfg.main_db.connection() as c:
key = None
if len(keys) == 1:
key = keys[0]
print "KEY derp", key
else:
# we need to do some basic set operations before proceeding
print "operator", operator
if operator == "intersect":
key = "TEMP_INTERSECT_ZSET"
c.zinterstore(key, keys)
elif operator == "union":
key = "TEMP_UNION_ZSET"
c.zunionstore(key, keys)
if count:
return c.zcard(key)
print "fetching %s:%s of %s reverse:%s" % (start, end, key, reverse)
if reverse:
post_ids = c.zrevrange(key, start, end)
else:
post_ids = c.zrange(key, start, end)
return post_ids
print __zset_fetch(
(
("hashtag", "science"),
("usage_type", "rekeep")
), operator="intersect", ids_only=True)
mget_rekeeps_by_hashtags = lambda hashtags, **kwargs: __zset_fetch(
[("usage_type", "rekeep"),] + [("hashtag", ht) for ht in hashtags],
operator="intersect", **kwargs)
print mget_rekeeps_by_hashtags(['science', 'derp'])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment