2018-09-19-zarr-mongodb.py

"""A MutableMapping interface to MongoDB that can be used with Zarr"""

import pymongo
import redis
from collections.abc import MutableMapping
import zarr

database_name = 'zarr'
collection_name = 'zarr'

client = pymongo.MongoClient()
coll = client[database_name]['zarr']


class MongoMap(MutableMapping):
    key_name = 'key'
    value_name = 'value'

    def __init__(self, coll):
        self.coll = coll

    def __getitem__(self, key):
        doc = self.coll.find_one({self.key_name: key})

        if doc is None:
            raise KeyError
        else:
            return doc[self.value_name]

    def __setitem__(self, key, val):
        del self[key]
        self.coll.insert_one({self.key_name: key, self.value_name: val})

    def __delitem__(self, key):
        self.coll.delete_many({self.key_name: key})

    def __iter__(self):
        for f in self.coll.find({}):
            yield f[self.key_name]

    def __len__(self):
        return self.coll.count_documents({})


class RedisStore(MutableMapping):


    def __init__(self, client, prefix='zarr'):
        self.client = client
        self.prefix = prefix


    def _key(self, key):
        return self.prefix + ':' + key

    def __getitem__(self, key):
        return self.client[self._key(key)]

    def __setitem__(self, key, val):
        self.client[self._key(key)] = val

    def __delitem__(self, key):
        del self.client[self._key(key)]

    def __iter__(self):
        for key in self.client.keys(self.prefix +":*"):
            yield key[len(self.prefix) + 1:].decode('utf-8')

    def __len__(self):
        return len(iter(self))

    def clear(self):
        for key in self.keys():
            del self[key]


m = MongoMap(coll)
r = redis.Redis()
r_store = RedisStore(r)

# for k in m:
#     print(k)

# print(len(m))
# m['a'] = b'adfd'
# print(m['a'])


zg = zarr.group(store=m)
# zg_r = zarr.group(store=r_store)
# r_store.clear()


import xarray as xr
# air = xr.tutorial.load_dataset('air_temperature')

# air.to_zarr(r_store)

b = xr.open_zarr(r_store)