Skip to content

Instantly share code, notes, and snippets.

@nanvel
Last active August 29, 2015 14:08
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save nanvel/8725b9c71c0040b0472b to your computer and use it in GitHub Desktop.
Save nanvel/8725b9c71c0040b0472b to your computer and use it in GitHub Desktop.
Get my friends activities using redis (redis JOIN alternative) code
import os.path
import sqlite3
import redis
import time
import uuid
class RedisSearch(object):
@property
def conn(self):
if hasattr(self, '_conn'):
return self._conn
self._conn = redis.StrictRedis(host='localhost')
return self._conn
def clean(self):
for key in self.conn.keys('test:*'):
self.conn.delete(key)
def add_friend(self, user, friend):
self.conn.sadd('test:friends:{user}'.format(user=user), friend)
def add_activity(self, user, activity, timestamp):
pipe = self.conn.pipeline()
pipe.zadd('test:last_user_activity', timestamp, user)
pipe.zadd('test:user_activities:{user}'.format(user=user), timestamp, activity)
pipe.execute()
def search(self, user, last, limit):
tmp_key = 'text:tmp:{user}'.format(user=user)
pipe = self.conn.pipeline(False)
pipe.zinterstore(
dest=tmp_key,
keys=['test:last_user_activity', 'test:friends:{user}'.format(user=user)],
aggregate='max')
pipe.zrevrange(tmp_key, 0, -1)
pipe.delete(tmp_key)
users = pipe.execute()[1]
if not users:
return []
user_keys = []
for u in users:
user_keys.append('test:user_activities:{user}'.format(user=u))
pipe = self.conn.pipeline(False)
pipe.zunionstore(dest=tmp_key, keys=user_keys, aggregate='max')
pipe.zremrangebyscore(tmp_key, min=last, max=get_timestamp())
pipe.zrevrange(tmp_key, 0, limit-1)
pipe.delete(tmp_key)
return pipe.execute()[2]
def get_timestamp():
return int(time.time() * 1000000)
if __name__ == '__main__':
db_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'activities.sqlite3')
con = sqlite3.connect(db_path)
redis_search = RedisSearch()
redis_search.clean()
with con:
cur = con.cursor()
cur.executescript(u"""
DROP TABLE IF EXISTS activities;
DROP TABLE IF EXISTS friends;
CREATE TABLE activities(id INTEGER PRIMARY KEY, user VARCHAR(31), activity VARCHAR(31), timestamp INTEGER);
CREATE TABLE friends(id INTEGER PRIMARY KEY, user VARCHAR(31), friend VARCHAR(31));
""")
authors = []
for i in xrange(100):
# create 100 activities
author = uuid.uuid4()
authors.append(author)
activity = uuid.uuid4()
timestamp = get_timestamp()
cur.executescript(u"""
INSERT INTO activities(user, activity, timestamp) VALUES("{user}", "{activity}", {timestamp});
""".format(user=author, activity=activity, timestamp=timestamp))
redis_search.add_activity(user=author, activity=activity, timestamp=timestamp)
user = uuid.uuid4()
for i in xrange(100):
# create friends
friend = uuid.uuid4()
cur.executescript(u"""
INSERT INTO friends(user, friend) VALUES("{user}", "{friend}");
""".format(user=user, friend=friend))
redis_search.add_friend(user=user, friend=friend)
# more friends
for i in xrange(100):
u = uuid.uuid4()
f = uuid.uuid4()
cur.executescript(u"""
INSERT INTO friends(user, friend) VALUES("{user}", "{friend}");
""".format(user=u, friend=f))
redis_search.add_friend(user=u, friend=f)
# add outhors to friends
for i in xrange(20):
cur.executescript(u"""
INSERT INTO friends(user, friend) VALUES("{user}", "{friend}");
""".format(user=user, friend=authors[i]))
redis_search.add_friend(user=user, friend=authors[i])
# select my friends activities
last = get_timestamp()
for i in xrange(2):
print '--- page {n} ---'.format(n=i + 1)
cur.execute(u"""
SELECT act.activity, act.timestamp from activities act
JOIN friends fr ON fr.friend=act.user AND fr.user="{user}"
WHERE act.timestamp < {last}
ORDER BY act.timestamp DESC
LIMIT {limit}
""".format(user=user, last=last, limit=10))
new_last = last
for r, timestamp in cur:
print r
new_last = timestamp
print '---'
for r in redis_search.search(user=user, last=last, limit=10):
print r
last = new_last
def search(self, user, last, limit):
SCRIPT = """
redis.call("ZINTERSTORE", "test:tmp:" .. ARGV[1], 2, "test:last_user_activity", "test:friends:" .. ARGV[1], "AGGREGATE", "MAX")
local users = redis.call("ZREVRANGE", "test:tmp:" .. ARGV[1], 0, -1)
if users == nil then
return {}
end
local keys = {}
table.insert(keys, "ZUNIONSTORE")
table.insert(keys, "test:tmp:" .. ARGV[1])
table.insert(keys, table.getn(users))
for k, v in pairs(users) do
table.insert(keys, "test:user_activities:" .. v)
end
table.insert(keys, "AGGREGATE")
table.insert(keys, "MAX")
redis.call(unpack(keys))
redis.call("ZREMRANGEBYSCORE", "test:tmp:" .. ARGV[1], ARGV[2], ARGV[3])
local users = redis.call("ZREVRANGE", "test:tmp:" .. ARGV[1], 0, ARGV[4] - 1)
redis.call("DEL", "test:tmp:" .. ARGV[1])
return users
"""
return self.conn.eval(SCRIPT, 0, user, last, get_timestamp(), limit)
def search(self, user, last, limit):
SCRIPT = """
redis.call("ZINTERSTORE", "test:tmp:" .. ARGV[1], 2, "test:last_user_activity", "test:friends:" .. ARGV[1], "AGGREGATE", "MAX")
local users = redis.call("ZREVRANGE", "test:tmp:" .. ARGV[1], 0, -1, "WITHSCORES")
if users == nil then
return {}
end
redis.call("DEL", "test:tmp:" .. ARGV[1])
local counter = 0
local lastval = users[1]
for k, v in pairs(users) do
if (counter % 2 == 0) then
lastval = v
else
redis.call("ZUNIONSTORE", "test:tmp:" .. ARGV[1], 2, "test:tmp:" .. ARGV[1], "test:user_activities:" .. lastval, "AGGREGATE", "MAX")
redis.call("ZREMRANGEBYSCORE", "test:tmp:" .. ARGV[1], ARGV[2], ARGV[3])
if redis.call("ZCOUNT", "test:tmp:" .. ARGV[1], v, ARGV[2]) >= tonumber(ARGV[4]) then break end
end
counter = counter + 1
end
local users = redis.call("ZREVRANGE", "test:tmp:" .. ARGV[1], 0, ARGV[4] - 1)
redis.call("DEL", "test:tmp:" .. ARGV[1])
return users
"""
return self.conn.eval(SCRIPT, 0, user, last, get_timestamp(), limit)
--- page 1 ---
efd363a1-226b-44e0-aa0c-02cb46c3a634
5738b144-41b9-498f-9cc2-8949ea3d1d17
dbd0172c-8a55-4642-b90b-980e7f823602
362ad0d4-44ab-4096-9240-79d5b2ee9f73
54e01af5-a003-4141-ac81-8fe78cb244ba
be168e73-c329-4dc3-8d9a-6d9926d0037c
ac7dbba7-b8c5-43cb-8748-486e6d181f86
3f86bfd6-c17d-495d-853d-fa4b3aa4f5d8
487253b3-b21c-4ca2-9cd3-c2e94ad0b617
b6ee615e-d04f-476b-921b-ed095ae3e75c
0:00:00.161883
---
efd363a1-226b-44e0-aa0c-02cb46c3a634
5738b144-41b9-498f-9cc2-8949ea3d1d17
dbd0172c-8a55-4642-b90b-980e7f823602
362ad0d4-44ab-4096-9240-79d5b2ee9f73
54e01af5-a003-4141-ac81-8fe78cb244ba
be168e73-c329-4dc3-8d9a-6d9926d0037c
ac7dbba7-b8c5-43cb-8748-486e6d181f86
3f86bfd6-c17d-495d-853d-fa4b3aa4f5d8
487253b3-b21c-4ca2-9cd3-c2e94ad0b617
b6ee615e-d04f-476b-921b-ed095ae3e75c
0:00:00.025598
--- page 2 ---
4cc6030a-499b-440b-a287-f68cc49927f8
97ad0261-9b91-4d71-9932-0158819749f6
ac1f1cd3-c530-4859-91a5-4392c9e1c2ad
84a16b15-dd94-4123-aac3-ff8f99cadc52
5a2eb1df-41c4-4e45-98dc-0cb998309167
f1e88d38-55aa-47a8-b8a9-b35339510df5
948cf405-dffc-4c5f-a4fb-1e34a30d4fae
dc3f8024-7ff5-4007-8915-d8ff0e82116a
24824183-dff0-434b-9f47-831974257df5
a8acb415-008c-46c2-8334-35d574089474
0:00:00.203902
---
4cc6030a-499b-440b-a287-f68cc49927f8
97ad0261-9b91-4d71-9932-0158819749f6
ac1f1cd3-c530-4859-91a5-4392c9e1c2ad
84a16b15-dd94-4123-aac3-ff8f99cadc52
5a2eb1df-41c4-4e45-98dc-0cb998309167
f1e88d38-55aa-47a8-b8a9-b35339510df5
948cf405-dffc-4c5f-a4fb-1e34a30d4fae
dc3f8024-7ff5-4007-8915-d8ff0e82116a
24824183-dff0-434b-9f47-831974257df5
a8acb415-008c-46c2-8334-35d574089474
0:00:00.026051
--- page 3 ---
d08e08d2-2fbe-4a8c-999c-2f824d0c39cc
5e833719-ad6d-4088-bb06-443d23e8cac3
35b5b296-71e1-4272-97e1-cf0f2673562a
00d873dc-6ee1-46ab-98d5-80d15721f59e
2b61d20b-a718-4621-9991-39e76fce9f85
0b2e8c04-27f7-4fd0-9f16-3d2133b624e8
fff83325-a650-4936-9374-c86ba923dcec
4cc8a41c-6ac3-450f-b075-ccd817cbbe25
75fd329b-d97f-4890-9202-d2533843c60e
e225c368-0a7d-479d-bc31-3f1d3004d0da
0:00:00.214251
---
d08e08d2-2fbe-4a8c-999c-2f824d0c39cc
5e833719-ad6d-4088-bb06-443d23e8cac3
35b5b296-71e1-4272-97e1-cf0f2673562a
00d873dc-6ee1-46ab-98d5-80d15721f59e
2b61d20b-a718-4621-9991-39e76fce9f85
0b2e8c04-27f7-4fd0-9f16-3d2133b624e8
fff83325-a650-4936-9374-c86ba923dcec
4cc8a41c-6ac3-450f-b075-ccd817cbbe25
75fd329b-d97f-4890-9202-d2533843c60e
e225c368-0a7d-479d-bc31-3f1d3004d0da
0:00:00.024963
--- page 4 ---
9ba7f259-e474-490e-9de0-05a8420d1f4f
643b36bc-3850-4ae5-baae-d1a371f01e92
47366aaa-55eb-4da7-9608-fe805f78ea7c
190f6a5b-f4eb-4bd9-83e7-68ff18a70f36
f81abe93-697b-4add-94e6-749216779c23
71b48821-711c-4af1-a70d-4fed16715e20
55113d0c-a8a5-40f0-aa57-d3c48e2d3f5e
29760629-a812-4a75-83f6-4bd39f865c14
585c469c-daf0-4479-8262-3ae0a9404a9e
6776bf22-d649-4a9a-90b7-1651532c4899
0:00:00.153230
---
9ba7f259-e474-490e-9de0-05a8420d1f4f
643b36bc-3850-4ae5-baae-d1a371f01e92
47366aaa-55eb-4da7-9608-fe805f78ea7c
190f6a5b-f4eb-4bd9-83e7-68ff18a70f36
f81abe93-697b-4add-94e6-749216779c23
71b48821-711c-4af1-a70d-4fed16715e20
55113d0c-a8a5-40f0-aa57-d3c48e2d3f5e
29760629-a812-4a75-83f6-4bd39f865c14
585c469c-daf0-4479-8262-3ae0a9404a9e
6776bf22-d649-4a9a-90b7-1651532c4899
0:00:00.032679
--- page 5 ---
efdfc3f4-1796-407a-9d09-365dd9f10c17
bd15d0b6-f11f-4898-9f50-a9a1c61ec6c7
c2511aaf-214b-4f9b-8548-593d26fc0615
16bd49e5-b5ba-450e-be93-b391f107a0b3
638e7c6d-80df-4374-902f-68118a1287ee
4896dd15-6f3b-4c02-b174-3582b5fed5ce
0c2e3c05-f83b-4c85-8b5e-cb05eb878663
3caefb07-8374-4f92-af7d-560d171fa6fd
c4695937-ffdd-40b3-99b9-dc1bc6cfcf21
a5562a2b-5b99-4cc3-b751-54dfd00408a3
0:00:00.177239
---
efdfc3f4-1796-407a-9d09-365dd9f10c17
bd15d0b6-f11f-4898-9f50-a9a1c61ec6c7
c2511aaf-214b-4f9b-8548-593d26fc0615
16bd49e5-b5ba-450e-be93-b391f107a0b3
638e7c6d-80df-4374-902f-68118a1287ee
4896dd15-6f3b-4c02-b174-3582b5fed5ce
0c2e3c05-f83b-4c85-8b5e-cb05eb878663
3caefb07-8374-4f92-af7d-560d171fa6fd
c4695937-ffdd-40b3-99b9-dc1bc6cfcf21
a5562a2b-5b99-4cc3-b751-54dfd00408a3
0:00:00.029828
--- page 6 ---
c622a45f-91f2-4b36-8708-cc68b49ff4d0
b14a2abc-c695-4326-a568-ca9a3d1557eb
eff9a7b3-89d5-4653-aa19-656788756ea2
3901d292-4360-484c-8037-4b3a5c24be65
5e69fc7b-64fe-495a-b359-a62871590ef5
4c1a296b-d735-4c29-97c3-b9f255a6f49e
799e1d8e-f78e-4013-9d76-80d97ec3f205
d4d1d263-89bb-48d2-9a86-d1eeefea7041
ee0a8d8c-c256-4398-89d1-eb99cabac6f4
9f297e5d-f2f6-41dc-992d-7c69db284cac
0:00:00.144073
---
c622a45f-91f2-4b36-8708-cc68b49ff4d0
b14a2abc-c695-4326-a568-ca9a3d1557eb
eff9a7b3-89d5-4653-aa19-656788756ea2
3901d292-4360-484c-8037-4b3a5c24be65
5e69fc7b-64fe-495a-b359-a62871590ef5
4c1a296b-d735-4c29-97c3-b9f255a6f49e
799e1d8e-f78e-4013-9d76-80d97ec3f205
d4d1d263-89bb-48d2-9a86-d1eeefea7041
ee0a8d8c-c256-4398-89d1-eb99cabac6f4
9f297e5d-f2f6-41dc-992d-7c69db284cac
0:00:00.035980
--- page 7 ---
f42da86d-af24-4f94-9b96-6b9b2d09cfee
af9788af-059c-4609-a66e-57f712d24293
5df79a44-d532-454e-8b42-55b5b115b98b
d7ba318a-cba7-40ae-bdff-5467572488cb
8743ebf6-0874-4953-a7c6-b83da0d8b69e
eb87c13f-76ed-4874-bc87-650c3dd4c9fa
9bfe14a5-ce9e-4760-a8b6-066031964d92
59ea0440-bf51-4875-90ab-c78a78a639b7
4c4faf1b-2255-4cbe-8365-e0b7091594d0
ada83c58-900c-49d5-9326-4d6328d1c53a
0:00:00.141307
---
f42da86d-af24-4f94-9b96-6b9b2d09cfee
af9788af-059c-4609-a66e-57f712d24293
5df79a44-d532-454e-8b42-55b5b115b98b
d7ba318a-cba7-40ae-bdff-5467572488cb
8743ebf6-0874-4953-a7c6-b83da0d8b69e
eb87c13f-76ed-4874-bc87-650c3dd4c9fa
9bfe14a5-ce9e-4760-a8b6-066031964d92
59ea0440-bf51-4875-90ab-c78a78a639b7
4c4faf1b-2255-4cbe-8365-e0b7091594d0
ada83c58-900c-49d5-9326-4d6328d1c53a
0:00:00.033097
--- page 8 ---
0ceb0b75-c9da-4363-b42c-b01b1373372f
b9d8d0b9-c150-41d9-9211-084d6def1a45
9cf51f5f-8a81-4825-8ab8-65821afaba9c
7814ffff-4b28-4805-9b07-ce01a0dc8fbd
4a5c0e44-2cb9-4317-a333-8b176a494aff
de6704a0-bb5f-4004-98bb-3555d570e719
efc640c5-b62b-42eb-8fb6-d19f4c8c7e81
40c7f481-19d7-47f7-ab03-e1bff2691ad9
f3953e2e-7afa-4d73-b295-821db22caad2
311f87f2-0f0e-4b42-8030-e2217ebd738a
0:00:00.174868
---
0ceb0b75-c9da-4363-b42c-b01b1373372f
b9d8d0b9-c150-41d9-9211-084d6def1a45
9cf51f5f-8a81-4825-8ab8-65821afaba9c
7814ffff-4b28-4805-9b07-ce01a0dc8fbd
4a5c0e44-2cb9-4317-a333-8b176a494aff
de6704a0-bb5f-4004-98bb-3555d570e719
efc640c5-b62b-42eb-8fb6-d19f4c8c7e81
40c7f481-19d7-47f7-ab03-e1bff2691ad9
f3953e2e-7afa-4d73-b295-821db22caad2
311f87f2-0f0e-4b42-8030-e2217ebd738a
0:00:00.038703
--- page 9 ---
6f484a95-8084-497b-a82c-64ab9fbf7d62
b0cf8659-f3df-48eb-8f4a-32cacbf5f206
2bfdedc1-8752-4809-8129-12ac87d1b035
1ea49a48-2d1b-4818-a7c8-d700f2a4b28e
28df1a8f-0225-47a9-8432-b79cc89afe4d
49507940-beda-4a51-8645-17bcc2848f16
9ff0d183-d17b-4f35-bfac-abfb08d1117f
515a9eee-3146-4b31-9d0a-2d3d36bff518
c35707ac-cc42-489a-b52e-365b7921670e
baa6c316-0f24-429f-ba9a-95b1017e6e67
0:00:00.179924
---
6f484a95-8084-497b-a82c-64ab9fbf7d62
b0cf8659-f3df-48eb-8f4a-32cacbf5f206
2bfdedc1-8752-4809-8129-12ac87d1b035
1ea49a48-2d1b-4818-a7c8-d700f2a4b28e
28df1a8f-0225-47a9-8432-b79cc89afe4d
49507940-beda-4a51-8645-17bcc2848f16
9ff0d183-d17b-4f35-bfac-abfb08d1117f
515a9eee-3146-4b31-9d0a-2d3d36bff518
c35707ac-cc42-489a-b52e-365b7921670e
baa6c316-0f24-429f-ba9a-95b1017e6e67
0:00:00.040392
--- page 10 ---
30773efb-75b8-4586-9be7-175b97d693e3
dfcbfa14-252c-44f6-bebf-47b402b0e6d1
cfc86f20-911c-43b2-b917-cf7d5b3cc7f4
763d1b52-943b-4edc-a799-8a4702596064
494503ec-eac1-4a0b-b1f1-815cb482c8e3
e737e636-d973-4275-8a5f-752d433aee59
7cdf176b-158f-488e-9edd-93210cb1577d
dae17dee-0c28-4512-bd88-5b03f4712d0c
1e4d18db-5857-4662-86f1-a6cf1c550609
fde63453-d404-49e6-8154-2734772c4cdd
0:00:00.149319
---
30773efb-75b8-4586-9be7-175b97d693e3
dfcbfa14-252c-44f6-bebf-47b402b0e6d1
cfc86f20-911c-43b2-b917-cf7d5b3cc7f4
763d1b52-943b-4edc-a799-8a4702596064
494503ec-eac1-4a0b-b1f1-815cb482c8e3
e737e636-d973-4275-8a5f-752d433aee59
7cdf176b-158f-488e-9edd-93210cb1577d
dae17dee-0c28-4512-bd88-5b03f4712d0c
1e4d18db-5857-4662-86f1-a6cf1c550609
fde63453-d404-49e6-8154-2734772c4cdd
0:00:00.048609
Activities count: 30000
Friends count: 25000
My friends count: 15000
Timings
posts: 10000
users: ~15000
friends: ~500
--- page 1 ---
69ab8e4c-05b8-49a4-bad5-eb7189a23d9a
fb18b582-a0d6-447f-b229-599ca07963e9
2aef1538-6864-42da-9efd-babe6a05b09a
2f4131ab-70d1-4aa7-94c4-67359bbbdee9
a7f51ccd-a661-413c-8706-2649433b303a
a624b926-975b-41fd-992a-c97555869711
ebd277e0-b26b-4b8c-8d76-85321985045e
8ad00885-07d6-4f22-95a9-c8663fb7970a
dadf3278-66f9-4ef0-ae2e-87da84ed1406
08ebc698-7fa6-41e2-a28b-a2bf42e2f572
0:00:00.051774
---
69ab8e4c-05b8-49a4-bad5-eb7189a23d9a
fb18b582-a0d6-447f-b229-599ca07963e9
2aef1538-6864-42da-9efd-babe6a05b09a
2f4131ab-70d1-4aa7-94c4-67359bbbdee9
a7f51ccd-a661-413c-8706-2649433b303a
a624b926-975b-41fd-992a-c97555869711
ebd277e0-b26b-4b8c-8d76-85321985045e
8ad00885-07d6-4f22-95a9-c8663fb7970a
dadf3278-66f9-4ef0-ae2e-87da84ed1406
08ebc698-7fa6-41e2-a28b-a2bf42e2f572
0:00:00.004863
--- page 2 ---
19f38295-195b-462f-911c-83397e818444
cc45b4dc-e244-43cd-9ffd-1a68cb488857
2f8f52aa-a737-4397-acd7-cb3db54b27be
bb495295-e604-4524-b697-dcb10c91addd
0e87b63c-7e73-4058-839f-c02972345c4f
9fa0ddd1-4484-4d93-864f-2d658733805f
eab3d509-b1a3-41ff-8ef9-7b01105de19a
8027629b-a76c-4923-b83b-86c77990566d
d5f18f5b-2819-4954-b793-548f91bc2b1d
97bd45f2-e3f5-4749-b39f-4092643955e4
0:00:00.057322
---
19f38295-195b-462f-911c-83397e818444
cc45b4dc-e244-43cd-9ffd-1a68cb488857
2f8f52aa-a737-4397-acd7-cb3db54b27be
bb495295-e604-4524-b697-dcb10c91addd
0e87b63c-7e73-4058-839f-c02972345c4f
9fa0ddd1-4484-4d93-864f-2d658733805f
eab3d509-b1a3-41ff-8ef9-7b01105de19a
8027629b-a76c-4923-b83b-86c77990566d
d5f18f5b-2819-4954-b793-548f91bc2b1d
97bd45f2-e3f5-4749-b39f-4092643955e4
0:00:00.004413
--- page 3 ---
4c460f26-d524-4fab-a68d-6bf229c3e1e4
b3d0b10d-bc5f-4552-b73d-862e05c277f3
640b9109-10b7-4c4c-958a-316d26593507
3307e048-99d4-4e53-9e68-fdb304e619ae
da60c1dc-c430-411e-b96a-d661c270312b
091cfbc2-23a1-4cea-8bbf-e58d782df98a
abde4e18-8386-40f9-a091-a5494b73aed1
2f78c384-0343-4507-ab87-912395d340a1
01c05e62-4a9c-4d96-b2ea-d137904dcaba
73963a5b-9e1f-439a-8f49-b93639375ad6
0:00:00.055446
---
4c460f26-d524-4fab-a68d-6bf229c3e1e4
b3d0b10d-bc5f-4552-b73d-862e05c277f3
640b9109-10b7-4c4c-958a-316d26593507
3307e048-99d4-4e53-9e68-fdb304e619ae
da60c1dc-c430-411e-b96a-d661c270312b
091cfbc2-23a1-4cea-8bbf-e58d782df98a
abde4e18-8386-40f9-a091-a5494b73aed1
2f78c384-0343-4507-ab87-912395d340a1
01c05e62-4a9c-4d96-b2ea-d137904dcaba
73963a5b-9e1f-439a-8f49-b93639375ad6
0:00:00.006977
--- page 4 ---
f3d439e1-1ece-49b9-b0e5-5f6590907635
37aec8ef-d939-4c3b-9602-67b6a05ca518
74b4cab8-346f-4c52-86a1-501f3da3c7da
cfe951b4-fa01-41b9-a21a-f0baee00d4c0
ca00fa23-860a-44a1-8fb1-966d099b9352
a01fbb28-c304-4dca-8c9d-8f7228b852b0
1622378e-5fd6-40c2-94a1-89ae0c5bf37c
0f29ad1b-a697-465f-9ee6-3ff2603e2df0
9730946a-0295-4355-829c-f7bd9c82b5f8
632a8523-ec67-4904-9423-deff8ab90d9f
0:00:00.055598
---
f3d439e1-1ece-49b9-b0e5-5f6590907635
37aec8ef-d939-4c3b-9602-67b6a05ca518
74b4cab8-346f-4c52-86a1-501f3da3c7da
cfe951b4-fa01-41b9-a21a-f0baee00d4c0
ca00fa23-860a-44a1-8fb1-966d099b9352
a01fbb28-c304-4dca-8c9d-8f7228b852b0
1622378e-5fd6-40c2-94a1-89ae0c5bf37c
0f29ad1b-a697-465f-9ee6-3ff2603e2df0
9730946a-0295-4355-829c-f7bd9c82b5f8
632a8523-ec67-4904-9423-deff8ab90d9f
0:00:00.010999
--- page 5 ---
995dae0f-3103-46fb-b1a2-5df3235ba4b6
0554d27f-9765-4679-b8f6-c2708d2252a4
76848a20-3798-4cee-8d9c-e415c2a4b832
53697da9-5a51-4ca8-9f9b-fb893f3c8d07
5a42bc57-43f6-4f9c-b3ba-675819c5af0b
8e0f8180-cf83-418f-938e-ef23b3b38d2a
cdd69750-6d9c-4a92-9547-c03d7b4facd5
a08def4a-0546-419d-907e-1a62abcec1fc
7e006d1d-9f5e-4457-a09e-453ee9b443c8
eb1cd648-cf35-4b6a-b638-c7c759967400
0:00:00.056296
---
995dae0f-3103-46fb-b1a2-5df3235ba4b6
0554d27f-9765-4679-b8f6-c2708d2252a4
76848a20-3798-4cee-8d9c-e415c2a4b832
53697da9-5a51-4ca8-9f9b-fb893f3c8d07
5a42bc57-43f6-4f9c-b3ba-675819c5af0b
8e0f8180-cf83-418f-938e-ef23b3b38d2a
cdd69750-6d9c-4a92-9547-c03d7b4facd5
a08def4a-0546-419d-907e-1a62abcec1fc
7e006d1d-9f5e-4457-a09e-453ee9b443c8
eb1cd648-cf35-4b6a-b638-c7c759967400
0:00:00.009230
--- page 6 ---
6690e530-b640-44ef-b10c-3f63ef1dca7f
6a9a0dcf-5b27-4237-b407-93ecb6bab31d
b5df2721-f94f-4c5b-b38e-943587f0546d
a2d50c41-11d7-4884-8a31-899e54cb720e
77a376e4-2711-4d27-80cb-d3b298f88a37
84e67566-ad22-4d78-b526-207a571c3f9f
5948db5d-40f7-4277-9549-0b08bf712a93
cc71124d-eb24-4ec2-bc0b-5b1ffbc0068b
3ef9ea65-e84e-40e1-919b-9f4b2672f2a9
dbcaf29c-695e-47d3-aa7f-fbd3b1a0816d
0:00:00.058927
---
6690e530-b640-44ef-b10c-3f63ef1dca7f
6a9a0dcf-5b27-4237-b407-93ecb6bab31d
b5df2721-f94f-4c5b-b38e-943587f0546d
a2d50c41-11d7-4884-8a31-899e54cb720e
77a376e4-2711-4d27-80cb-d3b298f88a37
84e67566-ad22-4d78-b526-207a571c3f9f
5948db5d-40f7-4277-9549-0b08bf712a93
cc71124d-eb24-4ec2-bc0b-5b1ffbc0068b
3ef9ea65-e84e-40e1-919b-9f4b2672f2a9
dbcaf29c-695e-47d3-aa7f-fbd3b1a0816d
0:00:00.011113
--- page 7 ---
58de1544-f31a-4767-ac5d-a5ef16ecc5c8
c155be07-45b5-4b0d-83d7-94de0b84762f
80bec62b-bb27-477b-a847-870ec2885060
77853e5f-02c3-4df1-a652-517c353c768a
aa9e1898-e897-410e-a84c-7cd65398c275
f8307556-a539-4b1b-8672-9f4ec9d34360
6aa204e6-7ead-40e4-9922-c82313d7dde1
34009fa0-e381-49a9-b98c-c96ff004e802
18f85450-8c17-4c14-9f3c-92b29c474568
174bfd21-dd46-4169-a5ec-4bd5549ca7e6
0:00:00.053850
---
58de1544-f31a-4767-ac5d-a5ef16ecc5c8
c155be07-45b5-4b0d-83d7-94de0b84762f
80bec62b-bb27-477b-a847-870ec2885060
77853e5f-02c3-4df1-a652-517c353c768a
aa9e1898-e897-410e-a84c-7cd65398c275
f8307556-a539-4b1b-8672-9f4ec9d34360
6aa204e6-7ead-40e4-9922-c82313d7dde1
34009fa0-e381-49a9-b98c-c96ff004e802
18f85450-8c17-4c14-9f3c-92b29c474568
174bfd21-dd46-4169-a5ec-4bd5549ca7e6
0:00:00.014001
--- page 8 ---
65df2c46-7702-45e4-8d7b-9f9e0a2cf2ba
e55c08c9-3acd-4b60-b816-9fb33fa74b64
a115e36d-7b4e-47ee-b0ff-96df05ea4e79
885692a2-b482-4c19-a57e-15a370d2665b
39fce91b-e5e7-48f9-854e-c0130f90a744
90e27292-2e42-4d37-9402-81568c480c24
203c5312-8962-4931-86e7-23870fb0b9b5
c0fe8a73-0955-4fb8-b986-6598d8afc9f3
0ee4f784-7ee6-418e-9ae4-4e11f411fe1d
b47a0638-6ac9-4648-9dd2-bdda249b0540
0:00:00.048016
---
65df2c46-7702-45e4-8d7b-9f9e0a2cf2ba
e55c08c9-3acd-4b60-b816-9fb33fa74b64
a115e36d-7b4e-47ee-b0ff-96df05ea4e79
885692a2-b482-4c19-a57e-15a370d2665b
39fce91b-e5e7-48f9-854e-c0130f90a744
90e27292-2e42-4d37-9402-81568c480c24
203c5312-8962-4931-86e7-23870fb0b9b5
c0fe8a73-0955-4fb8-b986-6598d8afc9f3
0ee4f784-7ee6-418e-9ae4-4e11f411fe1d
b47a0638-6ac9-4648-9dd2-bdda249b0540
0:00:00.023475
--- page 9 ---
de36a1a8-1ed5-4488-b279-27b36c379264
4da4ac68-a366-48bf-a984-49f89ea2b8f3
4f9410fb-30d4-4ee7-9f42-5c3643351626
dd86aac1-ed51-42a1-8210-bf9656a49510
89cfb092-2992-4426-9db5-019d25039839
9822b773-e92d-4b6a-9d44-82426ab2d871
d87bdbd0-f3e3-4a33-8bf4-cb4bcc51c191
49f703df-7e71-40f4-b49c-ee90297d9fa7
583c7a0d-24c3-4d1c-916c-afc95d8b70b0
39a75d31-fc8c-4afa-a627-dbb674b69bfa
0:00:00.052487
---
de36a1a8-1ed5-4488-b279-27b36c379264
4da4ac68-a366-48bf-a984-49f89ea2b8f3
4f9410fb-30d4-4ee7-9f42-5c3643351626
dd86aac1-ed51-42a1-8210-bf9656a49510
89cfb092-2992-4426-9db5-019d25039839
9822b773-e92d-4b6a-9d44-82426ab2d871
d87bdbd0-f3e3-4a33-8bf4-cb4bcc51c191
49f703df-7e71-40f4-b49c-ee90297d9fa7
583c7a0d-24c3-4d1c-916c-afc95d8b70b0
39a75d31-fc8c-4afa-a627-dbb674b69bfa
0:00:00.019728
--- page 10 ---
6ffaa423-5c14-46cf-85ef-7d48ae89b9f4
1bb0123f-cc9e-4cb0-8d84-48c445a92e82
f921ff4d-3d67-4239-9397-5ade41e20674
153a8c36-10d8-4078-a5bf-f0abe354442a
7b428ba7-d525-4e8d-acac-896076ae9752
13bf05cc-51c2-469a-a04c-f3389452331c
261e1834-03a2-4a52-aefa-07ab5fac48f3
e9096156-54d4-443c-9a3b-174e7c9a54b6
8416c96f-b889-40e7-9560-54ea554d0b17
0db24cfc-f875-4e28-8173-efb1b08a3c0d
0:00:00.052415
---
6ffaa423-5c14-46cf-85ef-7d48ae89b9f4
1bb0123f-cc9e-4cb0-8d84-48c445a92e82
f921ff4d-3d67-4239-9397-5ade41e20674
153a8c36-10d8-4078-a5bf-f0abe354442a
7b428ba7-d525-4e8d-acac-896076ae9752
13bf05cc-51c2-469a-a04c-f3389452331c
261e1834-03a2-4a52-aefa-07ab5fac48f3
e9096156-54d4-443c-9a3b-174e7c9a54b6
8416c96f-b889-40e7-9560-54ea554d0b17
0db24cfc-f875-4e28-8173-efb1b08a3c0d
0:00:00.022194
The first lua script version works much slower:
--- page 1 ---
6524f67a-90e4-49c1-8b85-c9ec16df0292
02fb160b-9eeb-4c72-9abe-6a9a71fd0f3d
9d358d7f-426d-4dfa-a495-75576b477856
9b2fb556-b8cd-41ed-929e-47afe69cb682
926b14e7-ddf8-4fb5-aa71-0633c3b0647e
03cec01a-6c98-4306-9876-d2dfa693cb13
b1b5bbdd-cbfa-43de-a3eb-3fa07174aa55
4fe4a699-1642-4626-85e3-49fccb231112
9f348183-199e-486f-b88a-e680009dbaa5
7f930fea-7821-40dd-9400-a391cd3ad987
0:00:00.115685
---
6524f67a-90e4-49c1-8b85-c9ec16df0292
02fb160b-9eeb-4c72-9abe-6a9a71fd0f3d
9d358d7f-426d-4dfa-a495-75576b477856
9b2fb556-b8cd-41ed-929e-47afe69cb682
926b14e7-ddf8-4fb5-aa71-0633c3b0647e
03cec01a-6c98-4306-9876-d2dfa693cb13
b1b5bbdd-cbfa-43de-a3eb-3fa07174aa55
4fe4a699-1642-4626-85e3-49fccb231112
9f348183-199e-486f-b88a-e680009dbaa5
7f930fea-7821-40dd-9400-a391cd3ad987
0:00:02.903959
import datetime
import os.path
import psycopg2
import random
import redis
import time
import uuid
CONN_STRING = "host='localhost' dbname='nanvel' user='nanvel' password='nanvel'"
class RedisSearch(object):
@property
def conn(self):
if hasattr(self, '_conn'):
return self._conn
self._conn = redis.StrictRedis(host='localhost')
return self._conn
def clean(self):
for key in self.conn.keys('test:*'):
self.conn.delete(key)
def add_friend(self, user, friend):
self.conn.sadd('test:friends:{user}'.format(user=user), friend)
def add_activity(self, user, activity, timestamp):
pipe = self.conn.pipeline()
pipe.zadd('test:last_user_activity', timestamp, user)
pipe.zadd('test:user_activities:{user}'.format(user=user), timestamp, activity)
pipe.execute()
def search(self, user, last, limit):
SCRIPT = """
redis.call("ZINTERSTORE", "test:tmp:" .. ARGV[1], 2, "test:last_user_activity", "test:friends:" .. ARGV[1], "AGGREGATE", "MAX")
local users = redis.call("ZREVRANGE", "test:tmp:" .. ARGV[1], 0, -1, "WITHSCORES")
if users == nil then
return {}
end
redis.call("DEL", "test:tmp:" .. ARGV[1])
local counter = 0
local lastval = users[1]
for k, v in pairs(users) do
if (counter % 2 == 0) then
lastval = v
else
redis.call("ZUNIONSTORE", "test:tmp:" .. ARGV[1], 2, "test:tmp:" .. ARGV[1], "test:user_activities:" .. lastval, "AGGREGATE", "MAX")
redis.call("ZREMRANGEBYSCORE", "test:tmp:" .. ARGV[1], ARGV[2], ARGV[3])
if redis.call("ZCOUNT", "test:tmp:" .. ARGV[1], v, ARGV[2]) >= tonumber(ARGV[4]) then break end
end
counter = counter + 1
end
local users = redis.call("ZREVRANGE", "test:tmp:" .. ARGV[1], 0, ARGV[4] - 1)
redis.call("DEL", "test:tmp:" .. ARGV[1])
return users
"""
return self.conn.eval(SCRIPT, 0, user, last, get_timestamp(), limit)
def get_timestamp():
return int(time.time() * 1000000)
if __name__ == '__main__':
conn = psycopg2.connect(CONN_STRING)
redis_search = RedisSearch()
redis_search.clean()
with conn:
cur = conn.cursor()
cur.execute(u"""
DROP TABLE IF EXISTS activities;
DROP TABLE IF EXISTS friends;
CREATE TABLE activities (
id SERIAL,
user_id VARCHAR(100),
activity_id VARCHAR(100),
timestamp BIGSERIAL
);
CREATE TABLE friends (
id SERIAL,
user_id VARCHAR(100),
friend_id VARCHAR(100)
);
CREATE INDEX activities_user_id_index ON activities (user_id);
CREATE INDEX activities_timestamp_index ON activities (timestamp);
CREATE INDEX friends_user_id_index ON friends (user_id);
CREATE INDEX friends_friend_id_index ON friends (friend_id);
""")
conn.commit()
authors = []
for i in xrange(30000):
# create 100 activities
if i % 2 == 0:
author = uuid.uuid4()
authors.append(author)
else:
author = random.choice(authors)
activity = uuid.uuid4()
timestamp = get_timestamp()
cur.execute(u"""
INSERT INTO activities(user_id, activity_id, timestamp) VALUES('{user}', '{activity}', {timestamp});
""".format(user=author, activity=activity, timestamp=timestamp))
redis_search.add_activity(user=author, activity=activity, timestamp=timestamp)
conn.commit()
user = uuid.uuid4()
for i in xrange(10000):
# create friends
friend = uuid.uuid4()
cur.execute(u"""
INSERT INTO friends(user_id, friend_id) VALUES('{user}', '{friend}');
""".format(user=user, friend=friend))
redis_search.add_friend(user=user, friend=friend)
conn.commit()
# more friends
for i in xrange(10000):
u = uuid.uuid4()
f = uuid.uuid4()
cur.execute(u"""
INSERT INTO friends(user_id, friend_id) VALUES('{user}', '{friend}');
""".format(user=u, friend=f))
redis_search.add_friend(user=u, friend=f)
conn.commit()
# add outhors to friends
for i in xrange(5000):
cur.execute(u"""
INSERT INTO friends(user_id, friend_id) VALUES('{user}', '{friend}');
""".format(user=user, friend=authors[i]))
redis_search.add_friend(user=user, friend=authors[i])
# select my friends activities
conn.commit()
time.sleep(1)
last = get_timestamp()
for i in xrange(10):
start = datetime.datetime.now()
print '--- page {n} ---'.format(n=i + 1)
cur.execute(u"""
SELECT act.activity_id, act.timestamp from activities act
JOIN friends fr ON fr.friend_id=act.user_id AND fr.user_id='{user}'
WHERE act.timestamp < {last}
ORDER BY act.timestamp DESC
LIMIT {limit}
""".format(user=user, last=last, limit=10))
new_last = last
for r, timestamp in cur:
print r
new_last = timestamp
print (datetime.datetime.now() - start)
print '---'
start = datetime.datetime.now()
for r in redis_search.search(user=user, last=last, limit=10):
print r
print (datetime.datetime.now() - start)
last = new_last
# stats
cur.execute(u"""SELECT count(*) from activities;""")
for i, in cur:
print 'Activities count:', i
cur.execute(u"""SELECT count(*) from friends;""")
for i, in cur:
print 'Friends count:', i
cur.execute(u"""SELECT count(*) from friends WHERE user_id='{user}';""".format(user=user))
for i, in cur:
print 'My friends count:', i
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment