-

#!/usr/bin/env python

import rocksdb
import requests
from urllib.parse import urlparse
from os.path import join
from json import loads

DB_PATH = "/mnt/data/tmp/spider/"
_DB = {}


def _db(name):
    db = _DB.get(name)
    if not db:
        db = _DB[name] = rocksdb.DB(
            join(DB_PATH, name), rocksdb.Options(create_if_missing=True)
        )
    return db


def get(url, *args, **kwds):
    db = _db(urlparse(url).netloc)
    _url = url.encode('utf-8')
    r = db.get(_url)
    if not 'timeout' in kwds:
        kwds['timeout'] = 60
    if not r:
        r = requests.get(url, *args, **kwds).content
        db.put(_url, r)
    return r


def get_json(url, *args, **kwds):
    r = get(url, *args, **kwds)
    return loads(r)