Skip to content

Instantly share code, notes, and snippets.

@ChrisCinelli
Last active November 20, 2018 00:09
Show Gist options
  • Save ChrisCinelli/ddae2b9db5f96db6e8a53059fe909e0e to your computer and use it in GitHub Desktop.
Save ChrisCinelli/ddae2b9db5f96db6e8a53059fe909e0e to your computer and use it in GitHub Desktop.
cache for npm request and elastic search
var _request = require('request');
const { stringify } = require('flatted/cjs');
const LRU = require('lru-cache');
const crypto = require('crypto');
// Interface for cached requests
class Cache {
// opt to pass to LRU, h hash function for the key
constructor({ opts = {}, h = (k) => k, Type = LRU }) {
this.cache = new Type(opts);
this.h = h;
}
// Return promisified object
getOrElse (key, valueF, opts = {}) {
const _key = this.h(key);
const cachedValue = this.cache.get(_key);
if (cachedValue) return cachedValue;
const pV = valueF();
this.cache.set(_key, pV, opts);
return pV;
}
}
function md5(obj) {
return crypto.createHash('MD5').update(stringify(obj)).digest('hex');
}
class CachedRequest {
constructor({ opts = {}}) {
this.cache = new Cache({ opts, h: md5});
this.opts = opts;
this.getCache = this.getCache.bind(this);
this.request = this.request.bind(this);
}
getCache() {
return this.cache;
}
request(url, cb) {
this.cache.getOrElse(url, () => {
return new Promise((resolve, reject) => {
//console.log('--------> Requesting', url),
_request(url, function(err, result, body) {resolve([err, result, body]);} );
})
}).then(r => cb.apply(this, r));
}
}
module.exports = {
md5,
Cache,
CachedRequest
};
const elasticsearch = require('elasticsearch');
const configHelper = require('./configHelper');
const { Cache, md5 } = require('./cachedRequest');
let esPromise;
let intervalId;
// When we need a ES connection we use getES()
// Both conenctions and searches are cached
const defaultOptions = {
esHost: 'esHost',
max: 10000, // max: max entries in the cache (default: 10000)
maxAge: undefined, // to set max cache age in ms (default: none)
cacheResetTime: 10 * 60 * 1000 // to set max cache age in ms (default: 10 minutes)
};
const pub = module.exports = {
opts: {
esHost: 'esHost',
max: 10000, // max: max entries in the cache (default: 10000)
maxAge: undefined, // to set max cache age in ms (default: none)
cacheResetTime: 10 * 60 * 1000 // to set max cache age in ms (default: 10 minutes)
},
setOpt: function(opts) {
pub.opts = {...defaultOptions, ...opts};
},
getES: function() {
if (!esPromise) {
esPromise = configHelper.getConfigValue(pub.opts.esHost).then(hostsFromConfig => {
let hosts = hostsFromConfig;
//Syntax for one host: username + ':' + password + '@' + host
if (typeof(hostsFromConfig) === 'string') hosts = [hosts];
var es = new elasticsearch.Client({
hosts
})
es.oldSearch = es.search;
const es_results_cache = new Cache({ h: md5, opts: {...pub.opts } });
es.search = function(what) {
return es_results_cache.getOrElse(what, () => {
// console.log('ES request:', JSON.stringify(what, null, 2))
return es.oldSearch(what)
});
}
return es;
});
}
return esPromise;
},
resetES: function () {
esPromise = undefined;
},
resetESInterval: function (time) {
if (intervalId) clearInterval(intervalId);
intervalId = setInterval(() => this.resetES(), pub.opts.cacheResetTime); // Default 10 minutes;
}
}
pub.resetESInterval();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment