mongodb 封装

mongo_cache.py

class MongoCache:
    db = None

    def __init__(self):
        if not hasattr(MongoCache, 'pool'):
            MongoCache.create_instance()

    @staticmethod
    def create_instance():
        client = MongoClient(config.MONGO_URL)
        MongoCache.db = client['spider']

    def create(self, table, unique_key, origin_data):
        if self.exists(table, unique_key):
            return None

        summaries = {k: generator_summary(v) for (k, v) in origin_data.items()}

        return self.db[table].insert({
            'unique_key': unique_key,
            'data': origin_data,
            'summaries': summaries
        })

    def get(self, table, unique_key):
        data = self.db[table].find_one({'unique_key': unique_key})
        if data is None:
            return None
        return data['data']

    def exists(self, table, unique_key):
        data = self.db[table].find_one({'unique_key': unique_key})
        return data is not None

    def is_changed(self, table, unique_key, origin_data):
        if not self.exists(table, unique_key):
            return True

        last_summaries = self.db[table].find_one({'unique_key': unique_key})['summaries']
        for (k, v) in origin_data.items():
            summary = generator_summary(v)
            last_summary = last_summaries.get(k, None)
            # print('{} -> {} | {} -> {}'.format(k, v, summary, last_summary))
            if last_summary is None or last_summary != summary:
                return True
        return False

    def change_fields(self, table, unique_key, origin_data):
        if not self.exists(table, unique_key):
            return origin_data
        changes = {}
        last_summaries = self.db[table].find_one({'unique_key': unique_key})['summaries']
        for (k, v) in origin_data.items():
            last_summary = last_summaries.get(k, None)
            # print('{} -> {} | {} -> {}'.format(k, v, summary, last_summary))
            if last_summary is None or last_summary != generator_summary(v):
                changes[k] = v
        return changes

    def update(self, table, unique_key, origin_data):
        if not self.exists(table, unique_key):
            return origin_data
        new_summaries = {k: generator_summary(v) for (k, v) in origin_data.items()}
        self.db[table].update_one({'unique_key': unique_key},
                                  {'$set': {'data': origin_data, 'summaries': new_summaries}})
        return origin_data