Skip to content

Instantly share code, notes, and snippets.

Created Jun 29, 2019

What would you like to do?
F2DB (Filesystem based Database, key-value store)


from hashlib import sha256
import pickle
import gzip
from pathlib import Path
import json
import pickle
import datetime
from concurrent.futures import ProcessPoolExecutor as PPE
import itertools
import re
import glob

class FFDB(object):
    def __init__(self, tar_path='tmp/ffdb'):
        self.tar_path = tar_path
        Path(self.tar_path).mkdir(exist_ok=True, parents=True)

    def get_hashed_fs(self, key):
        hashed = sha256(bytes(key, 'utf8')).hexdigest()[:16]
        fn = f'{self.tar_path}/{hashed}'
        return fn

    def exists(self, key):
        fn = self.get_hashed_fs(key)
        if Path(fn).exists():
            return True
        return False

    def save(self, key, val):
        fn = self.get_hashed_fs(key)
        with open(fn, 'wb') as fs:

    def get(self, key):
        fn = self.get_hashed_fs(key)
        if not Path(fn).exists():
            return None
        obj = None
        with open(fn, 'rb') as fs:
            obj = pickle.loads(gzip.decompress(
        return obj

    def get_iter(self):
        for fn in glob.glob(f'{self.tar_path}/*'):
            with open(fn, 'rb') as fp:
                obj = pickle.loads(gzip.decompress(
            yield obj
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.