Skip to content

Instantly share code, notes, and snippets.

@MOON-CLJ

MOON-CLJ/ iter_all_docs_1_3.diff

Last active December 28, 2015 21:29
Show Gist options
  • Save MOON-CLJ/7564556 to your computer and use it in GitHub Desktop.
Save MOON-CLJ/7564556 to your computer and use it in GitHub Desktop.
Download ZIP
本地database直接访问vs本地database tcp访问
Raw
iter_all_docs_1_3.diff
--- 1.txt 2013-11-20 21:24:35.000000000 +0800
+++ 3.txt 2013-11-20 21:24:35.000000000 +0800
@@ -1,4 +1,4 @@
- 309378992 function calls in 596.173 seconds
+ 309378996 function calls in 9818.898 seconds
Ordered by: standard name
@@ -33,7 +33,7 @@
1 0.000 0.000 0.000 0.000 __init__.py:2732(ExpandDecider)
1 0.000 0.000 0.000 0.000 __init__.py:2753(Enquire)
1 0.000 0.000 0.000 0.000 __init__.py:2770(__init__)
- 1 0.000 0.000 0.005 0.005 __init__.py:28(swig_import_helper)
+ 1 0.000 0.000 0.004 0.004 __init__.py:28(swig_import_helper)
1 0.000 0.000 0.000 0.000 __init__.py:280(InvalidArgumentError)
1 0.000 0.000 0.000 0.000 __init__.py:312(InvalidOperationError)
1 0.000 0.000 0.000 0.000 __init__.py:3164(Registry)
@@ -52,7 +52,7 @@
1 0.000 0.000 0.000 0.000 __init__.py:437(DatabaseCreateError)
1 0.001 0.001 0.003 0.003 __init__.py:44(<module>)
1 0.000 0.000 0.000 0.000 __init__.py:468(DatabaseLockError)
- 1 0.000 0.000 0.000 0.000 __init__.py:4768(open_stub)
+ 1 0.000 0.000 0.025 0.025 __init__.py:4902(remote_open)
1 0.000 0.000 0.000 0.000 __init__.py:4951(Query)
1 0.000 0.000 0.000 0.000 __init__.py:499(DatabaseModifiedError)
2 0.000 0.000 0.000 0.000 __init__.py:5038(__init__)
@@ -89,13 +89,13 @@
1 0.000 0.000 0.000 0.000 __init__.py:705(NetworkError)
1 0.000 0.000 0.000 0.000 __init__.py:71(ProcessError)
1 0.000 0.000 0.000 0.000 __init__.py:7157(PostingItem)
- 23798268 66.331 0.000 100.959 0.000 __init__.py:7175(__init__)
+ 23798268 231.899 0.000 3333.042 0.000 __init__.py:7175(__init__)
1 0.000 0.000 0.000 0.000 __init__.py:7217(PostingIter)
1 0.000 0.000 0.000 0.000 __init__.py:7226(__init__)
- 23798269 70.796 0.000 182.829 0.000 __init__.py:7242(next)
- 1 0.000 0.000 0.000 0.000 __init__.py:7286(_database_gen_postlist_iter)
+ 23798269 155.270 0.000 3512.102 0.000 __init__.py:7242(next)
+ 1 0.000 0.000 302.278 302.278 __init__.py:7286(_database_gen_postlist_iter)
1 0.000 0.000 0.000 0.000 __init__.py:7306(PositionIter)
- 23798268 9.724 0.000 9.724 0.000 __init__.py:7312(__init__)
+ 23798268 21.802 0.000 21.802 0.000 __init__.py:7312(__init__)
1 0.000 0.000 0.000 0.000 __init__.py:7351(ValueItem)
1 0.000 0.000 0.000 0.000 __init__.py:736(NetworkTimeoutError)
1 0.000 0.000 0.000 0.000 __init__.py:7367(ValueIter)
@@ -158,7 +158,7 @@
1 0.000 0.000 0.000 0.000 socket.py:167(_closedsocket)
1 0.000 0.000 0.000 0.000 socket.py:179(_socketobject)
1 0.000 0.000 0.000 0.000 socket.py:235(_fileobject)
- 1 0.002 0.002 0.002 0.002 socket.py:45(<module>)
+ 1 0.002 0.002 0.003 0.003 socket.py:45(<module>)
2 0.000 0.000 0.000 0.000 sre_compile.py:32(_compile)
2 0.000 0.000 0.000 0.000 sre_compile.py:361(_compile_info)
4 0.000 0.000 0.000 0.000 sre_compile.py:474(isstring)
@@ -177,7 +177,7 @@
2 0.000 0.000 0.000 0.000 sre_parse.py:90(__init__)
1 0.000 0.000 0.000 0.000 stat.py:24(S_IFMT)
1 0.000 0.000 0.000 0.000 stat.py:49(S_ISREG)
- 1 18.884 18.884 596.173 596.173 test_stub_file_open_tcp.py:3(<module>)
+ 1 48.901 48.901 9818.898 9818.898 test_stub_file_open_tcp.py:3(<module>)
1 0.000 0.000 0.001 0.001 threading.py:1(<module>)
1 0.000 0.000 0.000 0.000 threading.py:103(_RLock)
2 0.000 0.000 0.000 0.000 threading.py:180(Condition)
@@ -217,12 +217,13 @@
3 0.000 0.000 0.000 0.000 xapian_backend.py:118(fields_not_empty)
1 0.000 0.000 0.000 0.000 xapian_backend.py:119(_)
1 0.000 0.000 0.000 0.000 xapian_backend.py:127(XapianSearch)
- 1 0.000 0.000 0.000 0.000 xapian_backend.py:128(__init__)
- 23798269 54.364 0.000 577.264 0.000 xapian_backend.py:157(iter_all_docs)
+ 1 0.000 0.000 0.025 0.025 xapian_backend.py:128(__init__)
+ 23798269 177.237 0.000 9769.949 0.000 xapian_backend.py:157(iter_all_docs)
1 0.000 0.000 0.000 0.000 xapian_backend.py:18(Schema)
- 23798268 28.837 0.000 304.783 0.000 xapian_backend.py:227(_extract_item)
- 23798268 16.500 0.000 217.067 0.000 xapian_backend.py:281(_get_document_data)
- 1 0.000 0.000 0.000 0.000 xapian_backend.py:350(_stub_database)
+ 23798268 63.121 0.000 272.738 0.000 xapian_backend.py:227(_extract_item)
+ 23798268 41.960 0.000 81.998 0.000 xapian_backend.py:281(_get_document_data)
+ 1 0.000 0.000 0.025 0.025 xapian_backend.py:350(_stub_database)
+ 1 0.000 0.000 0.025 0.025 xapian_backend.py:366(create)
1 0.000 0.000 0.000 0.000 xapian_backend.py:380(InvalidIndexError)
1 0.002 0.002 0.024 0.024 xapian_backend.py:4(<module>)
2 0.000 0.000 0.000 0.000 {_sre.compile}
@@ -239,14 +240,14 @@
1 0.000 0.000 0.000 0.000 {_xapian.DatabaseModifiedError_swigregister}
1 0.000 0.000 0.000 0.000 {_xapian.DatabaseOpeningError_swigregister}
1 0.000 0.000 0.000 0.000 {_xapian.DatabaseVersionError_swigregister}
- 23798268 35.289 0.000 35.289 0.000 {_xapian.Database_get_document}
- 1 0.000 0.000 0.000 0.000 {_xapian.Database_postlist_begin}
+ 23798268 5505.594 0.000 5505.594 0.000 {_xapian.Database_get_document}
+ 1 302.278 302.278 302.278 302.278 {_xapian.Database_postlist_begin}
1 0.000 0.000 0.000 0.000 {_xapian.Database_postlist_end}
1 0.000 0.000 0.000 0.000 {_xapian.Database_swigregister}
1 0.000 0.000 0.000 0.000 {_xapian.DateValueRangeProcessor_swigregister}
1 0.000 0.000 0.000 0.000 {_xapian.DecreasingValueWeightPostingSource_swigregister}
1 0.000 0.000 0.000 0.000 {_xapian.DocNotFoundError_swigregister}
- 23798268 200.567 0.000 200.567 0.000 {_xapian.Document_get_data}
+ 23798268 40.038 0.000 40.038 0.000 {_xapian.Document_get_data}
1 0.000 0.000 0.000 0.000 {_xapian.Document_swigregister}
1 0.000 0.000 0.000 0.000 {_xapian.ESetIterator_swigregister}
1 0.000 0.000 0.000 0.000 {_xapian.ESet_swigregister}
@@ -274,10 +275,10 @@
1 0.000 0.000 0.000 0.000 {_xapian.NetworkTimeoutError_swigregister}
1 0.000 0.000 0.000 0.000 {_xapian.NumberValueRangeProcessor_swigregister}
1 0.000 0.000 0.000 0.000 {_xapian.PositionIterator_swigregister}
- 23798268 7.338 0.000 7.338 0.000 {_xapian.PostingIterator_get_docid}
- 23798268 11.969 0.000 11.969 0.000 {_xapian.PostingIterator_get_doclength}
- 23798268 5.597 0.000 5.597 0.000 {_xapian.PostingIterator_get_wdf}
- 23798268 11.074 0.000 11.074 0.000 {_xapian.PostingIterator_next}
+ 23798268 13.949 0.000 13.949 0.000 {_xapian.PostingIterator_get_docid}
+ 23798268 3032.417 0.000 3032.417 0.000 {_xapian.PostingIterator_get_doclength}
+ 23798268 32.975 0.000 32.975 0.000 {_xapian.PostingIterator_get_wdf}
+ 23798268 23.790 0.000 23.790 0.000 {_xapian.PostingIterator_next}
1 0.000 0.000 0.000 0.000 {_xapian.PostingIterator_swigregister}
1 0.000 0.000 0.000 0.000 {_xapian.PostingSource_swigregister}
1 0.000 0.000 0.000 0.000 {_xapian.QueryParserError_swigregister}
@@ -312,7 +313,7 @@
1 0.000 0.000 0.000 0.000 {_xapian.new_BoolWeight}
1 0.000 0.000 0.000 0.000 {_xapian.new_Enquire}
2 0.000 0.000 0.000 0.000 {_xapian.new_Query}
- 1 0.000 0.000 0.000 0.000 {_xapian.open_stub}
+ 1 0.025 0.025 0.025 0.025 {_xapian.remote_open}
2 0.000 0.000 0.000 0.000 {dir}
59 0.000 0.000 0.000 0.000 {getattr}
2 0.000 0.000 0.000 0.000 {hasattr}
@@ -321,6 +322,7 @@
15 0.000 0.000 0.000 0.000 {isinstance}
1 0.000 0.000 0.000 0.000 {issubclass}
79 0.000 0.000 0.000 0.000 {len}
+ 1 0.000 0.000 0.025 0.025 {map}
2 0.000 0.000 0.000 0.000 {method 'acquire' of 'thread.lock' objects}
209 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects}
2 0.000 0.000 0.000 0.000 {method 'close' of 'file' objects}
@@ -338,17 +340,18 @@
2 0.000 0.000 0.000 0.000 {method 'rfind' of 'str' objects}
2 0.000 0.000 0.000 0.000 {method 'rstrip' of 'str' objects}
5 0.000 0.000 0.000 0.000 {method 'setter' of 'property' objects}
- 2 0.000 0.000 0.000 0.000 {method 'split' of 'str' objects}
+ 3 0.000 0.000 0.000 0.000 {method 'split' of 'str' objects}
247 0.000 0.000 0.000 0.000 {method 'startswith' of 'str' objects}
51 0.000 0.000 0.000 0.000 {method 'strip' of 'str' objects}
42 0.000 0.000 0.000 0.000 {method 'update' of 'dict' objects}
4 0.000 0.000 0.000 0.000 {min}
- 23798268 58.879 0.000 58.879 0.000 {msgpack._unpacker.unpackb}
+ 23798268 127.620 0.000 127.620 0.000 {msgpack._unpacker.unpackb}
1 0.000 0.000 0.000 0.000 {open}
21 0.000 0.000 0.000 0.000 {ord}
2 0.000 0.000 0.000 0.000 {posix.getcwd}
1 0.000 0.000 0.000 0.000 {posix.stat}
1 0.000 0.000 0.000 0.000 {posix.urandom}
+ 1 0.000 0.000 0.000 0.000 {reduce}
14 0.000 0.000 0.000 0.000 {setattr}
3 0.000 0.000 0.000 0.000 {thread.allocate_lock}
2 0.000 0.000 0.000 0.000 {thread.get_ident}
Raw
gistfile1.diff
- 309378992 function calls in 596.173 seconds
+ 309378996 function calls in 9818.898 seconds
>>> 9818.898 / 596.173
- 23798268 66.331 0.000 100.959 0.000 __init__.py:7175(__init__)
+ 23798268 231.899 0.000 3333.042 0.000 __init__.py:7175(__init__)
class PostingItem(object):
def __init__(self, iter):
self._iter = iter
self.docid = iter._iter.get_docid()
self.doclength = iter._iter.get_doclength()
self.wdf = iter._iter.get_wdf()
# Support for sequence API
sequence = ['docid', 'doclength', 'wdf', 'positer']
if not iter._has_positions:
sequence[3] = PositionIter()
- 23798269 70.796 0.000 182.829 0.000 __init__.py:7242(next)
- 1 0.000 0.000 0.000 0.000 __init__.py:7286(_database_gen_postlist_iter)
+ 23798269 155.270 0.000 3512.102 0.000 __init__.py:7242(next)
+ 1 0.000 0.000 302.278 302.278 __init__.py:7286(_database_gen_postlist_iter)
class PostingIter(object):
def next(self):
if not self._moved:
self._iter.next()
self._moved = True
if self._iter == self._end:
raise StopIteration
else:
self._moved = False
return PostingItem(self)
def _database_gen_postlist_iter(self, tname):
if len(tname) != 0:
return PostingIter(self.postlist_begin(tname), self.postlist_end(tname),
has_positions=True)
else:
return PostingIter(self.postlist_begin(tname), self.postlist_end(tname))
Database.postlist = _database_gen_postlist_iter
- 23798268 9.724 0.000 9.724 0.000 __init__.py:7312(__init__)
+ 23798268 21.802 0.000 21.802 0.000 __init__.py:7312(__init__)
class PositionIter(object):
def __init__(self, start = 0, end = 0):
self.iter = start
self.end = end
- 23798269 54.364 0.000 577.264 0.000 xapian_backend.py:157(iter_all_docs)
+ 23798269 177.237 0.000 9769.949 0.000 xapian_backend.py:157(iter_all_docs)
def iter_all_docs(self, fields=None):
postlist_iter = db.postlist(match_all)
while 1:
try:
plitem = postlist_iter.next()
except StopIteration:
break
doc = db.get_document(plitem.docid)
yield self._extract_item(doc, fields)
def _extract_item(self, doc, fields):
r = msgpack.unpackb(self._get_document_data(self.database, doc))
- 23798268 28.837 0.000 304.783 0.000 xapian_backend.py:227(_extract_item)
- 23798268 16.500 0.000 217.067 0.000 xapian_backend.py:281(_get_document_data)
+ 23798268 63.121 0.000 272.738 0.000 xapian_backend.py:227(_extract_item)
+ 23798268 41.960 0.000 81.998 0.000 xapian_backend.py:281(_get_document_data)
- 23798268 58.879 0.000 58.879 0.000 {msgpack._unpacker.unpackb}
+ 23798268 127.620 0.000 127.620 0.000 {msgpack._unpacker.unpackb}
- 23798268 35.289 0.000 35.289 0.000 {_xapian.Database_get_document}
- 1 0.000 0.000 0.000 0.000 {_xapian.Database_postlist_begin}
+ 23798268 5505.594 0.000 5505.594 0.000 {_xapian.Database_get_document}
+ 1 302.278 302.278 302.278 302.278 {_xapian.Database_postlist_begin}
class Database(object):
def postlist_begin(self, *args):
return _xapian.Database_postlist_begin(self, *args)
def get_document(self, *args):
return _xapian.Database_get_document(self, *args)
def _database_gen_postlist_iter(self, tname):
Database.postlist = _database_gen_postlist_iter
- 23798268 200.567 0.000 200.567 0.000 {_xapian.Document_get_data}
+ 23798268 40.038 0.000 40.038 0.000 {_xapian.Document_get_data}
def _extract_item(self, doc, fields):
r = msgpack.unpackb(self._get_document_data(self.database, doc))
def _get_document_data(self, database, document):
try:
return document.get_data()
- 23798268 7.338 0.000 7.338 0.000 {_xapian.PostingIterator_get_docid}
- 23798268 11.969 0.000 11.969 0.000 {_xapian.PostingIterator_get_doclength}
- 23798268 5.597 0.000 5.597 0.000 {_xapian.PostingIterator_get_wdf}
- 23798268 11.074 0.000 11.074 0.000 {_xapian.PostingIterator_next}
+ 23798268 13.949 0.000 13.949 0.000 {_xapian.PostingIterator_get_docid}
+ 23798268 3032.417 0.000 3032.417 0.000 {_xapian.PostingIterator_get_doclength}
+ 23798268 32.975 0.000 32.975 0.000 {_xapian.PostingIterator_get_wdf}
+ 23798268 23.790 0.000 23.790 0.000 {_xapian.PostingIterator_next}
class PostingItem(object):
def __init__(self, iter):
self._iter = iter
self.docid = iter._iter.get_docid()
self.doclength = iter._iter.get_doclength()
self.wdf = iter._iter.get_wdf()
PostingIterator.get_doclength = new_instancemethod(_xapian.PostingIterator_get_doclength,None,PostingIterator)
PostingIterator.get_wdf = new_instancemethod(_xapian.PostingIterator_get_wdf,None,PostingIterator)
PostingIterator.get_docid = new_instancemethod(_xapian.PostingIterator_get_docid,None,PostingIterator)
PostingIterator.next = new_instancemethod(_xapian.PostingIterator_next,None,PostingIterator)
Raw
gistfile2.diff
- 23798268 66.331 0.000 100.959 0.000 __init__.py:7175(__init__)
+ 23798268 231.899 0.000 3333.042 0.000 __init__.py:7175(__init__)
class PostingItem(object):
def __init__(self, iter):
self._iter = iter
self.docid = iter._iter.get_docid()
self.doclength = iter._iter.get_doclength()
self.wdf = iter._iter.get_wdf()
# Support for sequence API
sequence = ['docid', 'doclength', 'wdf', 'positer']
if not iter._has_positions:
sequence[3] = PositionIter()
- 1 0.000 0.000 0.000 0.000 __init__.py:7286(_database_gen_postlist_iter)
+ 1 0.000 0.000 302.278 302.278 __init__.py:7286(_database_gen_postlist_iter)
class PostingIter(object):
def next(self):
if not self._moved:
self._iter.next()
self._moved = True
if self._iter == self._end:
raise StopIteration
else:
self._moved = False
return PostingItem(self)
def _database_gen_postlist_iter(self, tname):
if len(tname) != 0:
return PostingIter(self.postlist_begin(tname), self.postlist_end(tname),
has_positions=True)
else:
return PostingIter(self.postlist_begin(tname), self.postlist_end(tname))
Database.postlist = _database_gen_postlist_iter
- 23798268 9.724 0.000 9.724 0.000 __init__.py:7312(__init__)
+ 23798268 21.802 0.000 21.802 0.000 __init__.py:7312(__init__)
class PositionIter(object):
def __init__(self, start = 0, end = 0):
self.iter = start
self.end = end
def iter_all_docs(self, fields=None):
postlist_iter = db.postlist(match_all)
while 1:
try:
plitem = postlist_iter.next()
except StopIteration:
break
doc = db.get_document(plitem.docid)
yield self._extract_item(doc, fields)
def _extract_item(self, doc, fields):
r = msgpack.unpackb(self._get_document_data(self.database, doc))
- 23798268 58.879 0.000 58.879 0.000 {msgpack._unpacker.unpackb}
+ 23798268 127.620 0.000 127.620 0.000 {msgpack._unpacker.unpackb}
- 23798268 35.289 0.000 35.289 0.000 {_xapian.Database_get_document}
+ 23798268 5505.594 0.000 5505.594 0.000 {_xapian.Database_get_document}
class Database(object):
def postlist_begin(self, *args):
return _xapian.Database_postlist_begin(self, *args)
def get_document(self, *args):
return _xapian.Database_get_document(self, *args)
def _database_gen_postlist_iter(self, tname):
Database.postlist = _database_gen_postlist_iter
- 23798268 200.567 0.000 200.567 0.000 {_xapian.Document_get_data}
+ 23798268 40.038 0.000 40.038 0.000 {_xapian.Document_get_data}
def _extract_item(self, doc, fields):
r = msgpack.unpackb(self._get_document_data(self.database, doc))
def _get_document_data(self, database, document):
try:
return document.get_data()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment