Skip to content

Instantly share code, notes, and snippets.

@jborg
Created May 2, 2014 19:00
Show Gist options
  • Save jborg/7b633289bf5521173749 to your computer and use it in GitHub Desktop.
Save jborg/7b633289bf5521173749 to your computer and use it in GitHub Desktop.
Reduce attic file cache memory usage
diff --git a/attic/cache.py b/attic/cache.py
index 65362ff..6368005 100644
--- a/attic/cache.py
+++ b/attic/cache.py
@@ -12,6 +12,8 @@ from .hashindex import ChunkIndex
class Cache(object):
"""Client Side cache
"""
+ # Do not cache file metadata for files smaller than this
+ FILE_MIN_SIZE = 4096
class RepositoryReplay(Error):
"""Cache is newer than repository, refusing to continue"""
@@ -81,9 +83,10 @@ class Cache(object):
if not data:
break
u.feed(data)
- for hash, item in u:
+ for path_hash, item in u:
+ if item[2] > self.FILE_MIN_SIZE:
item[0] += 1
- self.files[hash] = item
+ self.files[path_hash] = item
def begin_txn(self):
# Initialize transaction snapshot
@@ -218,7 +221,8 @@ class Cache(object):
return None
def memorize_file(self, path_hash, st, ids):
- # Entry: Age, inode, size, mtime, chunk ids
- mtime_ns = st_mtime_ns(st)
- self.files[path_hash] = 0, st.st_ino, st.st_size, mtime_ns, ids
- self._newest_mtime = max(self._newest_mtime, mtime_ns)
+ if st.st_size > self.FILE_MIN_SIZE:
+ # Entry: Age, inode, size, mtime, chunk ids
+ mtime_ns = st_mtime_ns(st)
+ self.files[path_hash] = 0, st.st_ino, st.st_size, mtime_ns, ids
+ self._newest_mtime = max(self._newest_mtime, mtime_ns)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment