Skip to content

Instantly share code, notes, and snippets.

@kixxauth
Last active March 24, 2017 11:36
Show Gist options
  • Save kixxauth/14c58177aed3a1974d412b10133d3816 to your computer and use it in GitHub Desktop.
Save kixxauth/14c58177aed3a1974d412b10133d3816 to your computer and use it in GitHub Desktop.
Oddworks Datastore Scan Tests
'use strict';
const Promise = require('bluebird');
const _ = require('lodash');
const redis = require('redis');
const REDIS_SCAN_PAGES = 5;
const PAGE_SIZE = 20;
Promise.promisifyAll(redis.RedisClient.prototype);
Promise.promisifyAll(redis.Multi.prototype);
exports.main = function (args) {
const url = args.redisUrl;
const client = redis.createClient({url});
const PATTERN = 'crtv:video:*';
function getPage(pages, results, cursor, pageSize) {
return client.scanAsync(cursor, 'MATCH', PATTERN, 'COUNT', pageSize).then(res => {
pages += 1;
results = results.concat(res[1]);
cursor = parseInt(res[0], 10);
if (cursor) {
return getPage(pages, results, cursor, pageSize);
}
return {pages, results};
});
}
function hscan(keys) {
const commands = keys.map(key => {
return ['hscan', key, 0, 'COUNT', 256];
});
return client.multi(commands).execAsync().then(results => {
return _.flatten(results.map(res => {
return res[1].filter((item, i) => {
return i % 2 === 1;
});
}));
});
}
const start = Date.now();
let pageSize;
let keys;
let redisPages;
let results;
console.log(' --- start ---');
return Promise.resolve(null)
.then(() => {
return client.dbsizeAsync().then(res => {
pageSize = Math.ceil(res / REDIS_SCAN_PAGES);
return null;
});
})
.then(() => {
return getPage(0, [], 0, pageSize).then(res => {
keys = res.results;
redisPages = res.pages;
return null;
});
})
.then(() => {
const time = Date.now() - start;
console.log('Pattern:', PATTERN);
console.log('Total keys:', keys.length);
console.log('Total Redis pages required:', redisPages);
console.log('Redis Page size', pageSize);
console.log('Time required to get keys:', time);
return null;
})
.then(() => {
return hscan(keys.slice(0, PAGE_SIZE)).then(items => {
results = JSON.parse(`[${items.join(',')}]`);
});
})
.then(() => {
const time = Date.now() - start;
console.log('Page size', PAGE_SIZE);
console.log('Results:', results.length);
console.log('Time required to get results:', time);
})
.then(() => {
client.end(false);
return Promise.delay(300);
});
};
if (require.main === module) {
const redisUrl = process.env.REDIS_URL;
if (!redisUrl) {
throw new Error('REDIS_URL env var must be set');
}
exports.main({redisUrl}).then(() => {
console.log(' --- done ---');
}).catch(err => {
console.log(' --- done; with error ---');
console.error(err.stack);
});
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment