Skip to content

Instantly share code, notes, and snippets.

@wmantly
Last active Jun 25, 2021
Embed
What would you like to do?
Simple redis ORM for nodeJS. All code is MIT licensed.
# William Mantly
# MIT
'use strict';
const process_type = {
number: function(key, value){
if(key.min && value < key.min) return `is to small, min ${key.min}.`
if(key.max && value > key.max) return `is to large, max ${key.max}.`
},
string: function(key, value){
if(key.min && value.length < key.min) return `is too short, min ${key.min}.`
if(key.max && value.length > key.max) return `is too short, max ${key.max}.`
},
}
function returnOrCall(value){
return typeof(value) === 'function' ? value() : value;
}
function processKeys(map, data, partial){
let errors = [];
let out = {};
for(let key of Object.keys(map)){
if(!map[key].always && partial && !data.hasOwnProperty(key)) continue;
if(!partial && map[key].isRequired && !data.hasOwnProperty(key)){
errors.push({key, message:`${key} is required.`});
continue;
}
if(data.hasOwnProperty(key) && map[key].type && typeof(data[key]) !== map[key].type){
errors.push({key, message:`${key} is not ${map[key].type} type.`});
continue;
}
out[key] = data.hasOwnProperty(key) && data[key] !== undefined ? data[key] : returnOrCall(map[key].default);
if(data.hasOwnProperty(key) && process_type[map[key].type]){
let typeError = process_type[map[key].type](map[key], data[key]);
if(typeError){
errors.push({key, message:`${key} ${typeError}`});
continue;
}
}
}
if(errors.length !== 0){
throw new ObjectValidateError(errors);
return {__errors__: errors};
}
return out;
}
function parseFromString(map, data){
let types = {
boolean: function(value){ return value === 'false' ? false : true },
number: Number,
string: String,
object: JSON.parse
};
for(let key of Object.keys(data)){
if(map[key] && map[key].type){
data[key] = types[map[key].type](data[key]);
}
}
return data;
}
function parseToString(data){
let types = {
object: JSON.stringify
}
return (types[typeof(data)] || String)(data);
}
function ObjectValidateError(message) {
this.name = 'ObjectValidateError';
this.message = (message || {});
this.status = 422;
}
ObjectValidateError.prototype = Error.prototype;
module.exports = {processKeys, parseFromString, ObjectValidateError, parseToString};
# William Mantly
# MIT
'use strict';
const {createClient} = require('redis');
const {promisify} = require('util');
const config = {
prefix: 'deploy_'
}
function client() {
return createClient(config);
}
const _client = client();
const SCAN = promisify(_client.SCAN).bind(_client);
module.exports = {
client: client,
HGET: promisify(_client.HGET).bind(_client),
HDEL: promisify(_client.HDEL).bind(_client),
SADD: promisify(_client.SADD).bind(_client),
SREM: promisify(_client.SREM).bind(_client),
DEL: promisify(_client.DEL).bind(_client),
HSET: promisify(_client.HSET).bind(_client),
HGETALL: promisify(_client.HGETALL).bind(_client),
SMEMBERS: promisify(_client.SMEMBERS).bind(_client),
RENAME: promisify(_client.RENAME).bind(_client),
HSCAN: promisify(_client.HSCAN).bind(_client),
SCAN: async function(match){
let coursor = 0;
let results = [];
do{
let res = await SCAN(coursor, 'MATCH', config.prefix+match);
coursor = Number(res[0]);
results.push(...res[1].map(e => e.replace(config.prefix, '')))
} while(coursor);
return results
}
};
# William Mantly
# MIT
'use strict';
const client = require('../utils/redis');
const objValidate = require('../utils/object_validate');
class Table{
constructor(data){
for(let key in data){
this[key] = data[key];
}
}
static async get(index){
try{
let result = await client.HGETALL(`${this.prototype.constructor.name}_${index}`);
if(!result){
let error = new Error('EntryNotFound');
error.name = 'EntryNotFound';
error.message = `${this.prototype.constructor.name}:${index} does not exists`;
error.status = 404;
throw error;
}
// Redis always returns strings, use the keyMap schema to turn them
// back to native values.
result = objValidate.parseFromString(this._keyMap, result);
return new this.prototype.constructor(result)
}catch(error){
throw error;
}
}
static async exists(index){
try{
await this.get(data);
return true
}catch(error){
return false;
}
}
static async list(){
// return a list of all the index keys for this table.
try{
return await client.SMEMBERS(this.prototype.constructor.name);
}catch(error){
throw error;
}
}
static async listDetail(){
// Return a list of the entries as instances.
let out = [];
for(let entry of await this.list()){
out.push(await this.get(entry));
}
return out
}
static async add(data){
// Add a entry to this redis table.
try{
// Validate the passed data by the keyMap schema.
data = objValidate.processKeys(this._keyMap, data);
// Do not allow the caller to overwrite an existing index key,
if(data[this._key] && await this.exists(data)){
let error = new Error('EntryNameUsed');
error.name = 'EntryNameUsed';
error.message = `${this.prototype.constructor.name}:${data[this._key]} already exists`;
error.status = 409;
throw error;
}
// Add the key to the members for this redis table
await client.SADD(this.prototype.constructor.name, data[this._key]);
// Add the values for this entry.
for(let key of Object.keys(data)){
await client.HSET(`${this.prototype.constructor.name}_${data[this._key]}`, key, objValidate.parseToString(data[key]));
}
// return the created redis entry as entry instance.
return await this.get(data[this._key]);
} catch(error){
throw error;
}
}
async update(data, key){
// Update an existing entry.
try{
// Check to see if entry name changed.
if(data[this.constructor._key] && data[this.constructor._key] !== this[this.constructor._key]){
// Merge the current data into with the updated data
let newData = Object.assign({}, this, data);
// Remove the updated failed so it doesnt keep it
delete newData.updated;
// Create a new record for the updated entry. If that succeeds,
// delete the old recored
if(await this.add(newData)) await this.remove();
}else{
// Update what ever fields that where passed.
// Validate the passed data, ignoring required fields.
data = objValidate.processKeys(this.constructor._keyMap, data, true);
// Loop over the data fields and apply them to redis
for(let key of Object.keys(data)){
this[key] = data[key];
await client.HSET(`${this.constructor.name}_${this[this.constructor._key]}`, key, data[key]);
}
}
return this;
} catch(error){
// Pass any error to the calling function
throw error;
}
}
async remove(data){
// Remove an entry from this table.
try{
// Remove the index key from the tables members list.
await client.SREM(this.constructor.name, this[this.constructor._key]);
// Remove the entries hash values.
let count = await client.DEL(`${this.constructor.name}_${this[this.constructor._key]}`);
// Return the number of removed values to the caller.
return count;
} catch(error) {
throw error;
}
};
}
module.exports = Table;
'use strict';
const {promisify} = require('util');
const forge = require('node-forge');
const Table = require('../utils/redis_model');
var rasGenerate = promisify(forge.pki.rsa.generateKeyPair);
async function generateOpenSshPair(keySize){
keySize = keySize || 2048;
let keyPair = await rasGenerate({bits: keySize});
return {
publicKey: forge.ssh.publicKeyToOpenSSH(keyPair.publicKey),
privateKey: forge.ssh.privateKeyToOpenSSH(keyPair.privateKey)
};
};
const UUID = function b(a){return a?(a^Math.random()*16>>a/4).toString(16):([1e7]+-1e3+-4e3+-8e3+-1e11).replace(/[018]/g,b)};
class Repo extends Table{
static _key = 'repo'
static _keyMap = {
'created_by': {isRequired: true, type: 'string', min: 3, max: 500},
'created_on': {default: function(){return (new Date).getTime()}},
'updated_by': {default:"__NONE__", isRequired: false, type: 'string',},
'updated_on': {default: function(){return (new Date).getTime()}, always: true},
'repo': {isRequired: true, type: 'string', min: 3, max: 500},
'hookCallCount': {default: 0, type: 'number'},
'scriptsPath': {default:'scripts', type: 'string'},
'settings': {default: {}, type:'object'},
'secrets': {default: {}, type: 'object', min: 3, max: 500},
'privateKey': {type: 'string'},
'publicKey': {type: 'string'},
}
constructor(...args){
super(...args);
}
static async add(data){
return super.add({...data, ...(await generateOpenSshPair(2048))})
}
async getEnvironments(){
let environments = await Environment.list();
let out = [];
for(let environment of environments){
if(environment.startsWith(this.repo)){
environment = await Environment.get(environment);
environment.repo = this;
out.push(environment)
}
}
return out;
}
async getEnvironmentsbyBranch(branch){
let list = await this.getEnvironments();
let any;
for(let key of list){
if(branch === key.branchMatch) return key;
if(key.branchMatch === '*') any = key;
}
return any;
}
async getDeploymentsbyBranch(branch, state){
let environment = await this.getEnvironmentsbyBranch(branch);
let deployments = await Deployment.list();
let out = []
for(let deployment of deployments){
if(deployment.startsWith(`${this.repo}_${environment.environment}`)){
deployment = await Deployment.get(deployment);
deployment.environment = environment;
deployment.target = await Target.get(environment.target);
out.push(deployment)
if(state && deployment.state === state){
}
}
}
return out;
}
}
class Environment extends Table{
static _key = 'repo_env'
static _keyMap = {
'created_by': {isRequired: true, type: 'string', min: 3, max: 500},
'created_on': {default: function(){return (new Date).getTime()}},
'updated_by': {default:"__NONE__", isRequired: false, type: 'string',},
'updated_on': {default: function(){return (new Date).getTime()}, always: true},
'repo_env': {isRequired: true, type: 'string', min: 3, max: 500},
'repo': {type: 'string', min: 3, max: 500},
'environment': {isRequired: true, type: 'string', min: 3, max: 500},
'branchMatch': {isRequired: true, type: 'string', min: 1, max: 500},
'target': {isRequired: true, type: 'string', min: 3, max: 500},
'settings': {default: {}, type: 'object', min: 3, max: 500},
'secrets': {default: {}, type: 'object', min: 3, max: 500},
'hookCallCount': {default: 0, type: 'number'},
'lastCommit': {default:"__NONE__", isRequired: false, type: 'string'},
'workingPath': {default: '/opt/datacom', type: 'string'},
'domain': {isRequired: true, type: 'string'},
}
static async add(data){
try{
await Repo.get(data.repo);
await Target.get(data.target);
data.repo_env = `${data.repo}_${data.environment}`
return await super.add(data);
}catch(error){
throw error;
}
};
async addDeployment(data){
try{
data = data || {}
data.created_by = data.uid || this.created_by;
data.repo = this.repo.repo || this.repo;
data.environment = this.environment;
data.id = UUID().split('-').reverse()[0]
data.repo_env_id = `${data.repo}_${data.environment}_${data.id}`
let deployment = await Deployment.add(data);
deployment.target = await Target.get(this.target)
deployment.environment = this;
return deployment;
}catch(error){
throw error;
}
};
}
class Deployment extends Table{
static _key = 'repo_env_id'
static _keyMap = {
'created_by': {isRequired: true, type: 'string', min: 3, max: 500},
'created_on': {default: function(){return (new Date).getTime()}},
'updated_by': {default:"__NONE__", isRequired: false, type: 'string',},
'updated_on': {default: function(){return (new Date).getTime()}, always: true},
'id': {type: 'string', min: 12, max: 12},
'repo_env_id': {isRequired: true, type: 'string', min: 3, max: 500},
'repo': {type: 'string', min: 3, max: 500},
'environment': {isRequired: true, type: 'string', min: 3, max: 500},
'state': {default: 'new', type: 'string', min: 3, max: 500},
'isActive': {default: true, type: 'boolean',},
'target_url': {default:"__NONE__", isRequired: false, type: 'string'},
}
}
class Target extends Table{
static _key = 'name'
static _keyMap = {
'created_by': {isRequired: true, type: 'string', min: 3, max: 500},
'created_on': {default: function(){return (new Date).getTime()}},
'updated_by': {default:"__NONE__", isRequired: false, type: 'string',},
'updated_on': {default: function(){return (new Date).getTime()}, always: true},
'name': {isRequired: true, type: 'string', min: 2, max: 500},
'type': {isRequired: true, type: 'string', min: 1, max: 36},
'settings': {default: {}, type: 'object', min: 3, max: 500},
}
}
module.exports = {Repo, Environment, Deployment, Target};
(async function(){try{
// // console.log(await Repo.list())
// // To ssh://git.theta42.com:2222/wmantly/static-test.git
let lxc_starting = await Target.add({
created_by: 'wmantly',
name: 'lxc_starting',
type: 'LXC',
settings: {
user:'virt-service',
host:'lxc-staging0.sfo2.do.datacominfra.net',
keyPath:'/home/william/.ssh/id_rsa_virt-service'
}
});
var repo = await Repo.add({
created_by: 'wmantly',
repo: 'wmantly/static-test',
})
var environment = await Environment.add({
created_by: 'wmantly',
environment: 'staging',
branchMatch: '*',
repo: 'wmantly/static-test',
domain: '*.dc.vm42.us',
target: 'lxc_starting'
})
// let environment = await Environment.get('wmantly/static-test_staging')
// await environment.update({'domain': '*.dc.vm42.us'})
// // console.log(test)
// // console.log(await Environment.listDetail())
// // let repo = await Repo.get('wmantly/test2')
// // console.log(repo)
// // repo.update({hookCallCount: 5});
// // let envs = await repo.getEnvironments();
// // let env = await repo.getEnvironmentsbyBranch('staging');
// // let deployment = await env.addDeployment()
// // console.log('deployment', deployment)
// // let deployments = await repo.getDeploymentsbyBranch('staging')
// // console.log('deployments', deployments)
// // console.log('deployments', await Deployment.listDetail())
// console.log('repo', await Repo.listDetail())
// console.log('environment', await Environment.listDetail())
// for(let d of await Deployment.listDetail()){
// console.log('to remove', d)
// await d.remove()
// }
// console.log('deployment', await Deployment.listDetail())
// console.log('blah')
// let repo = await Repo.get('wmantly/static-test');
// // let environment = await repo.getEnvironmentsbyBranch('master')
// // console.log('environment', environment)
// let deployment = await repo.getDeploymentsbyBranch('master')
// console.log('deployments', deployment)
// return 0;
}catch(error){
console.error('IIFE error', error, error.message);
}})()
'use strict';
const router = require('express').Router();
const {Repo} = require('../models/repo');
const Model = Repo;
router.get('/', async function(req, res, next){
try{
return res.json({
hosts: await Model[req.query.detail ? "listDetail" : "list"]()
});
}catch(error){
return next(error);
}
});
router.post('/', async function(req, res, next){
try{
req.body.created_by = req.user.username;
await Model.add(req.body);
return res.json({
message: `"${req.body.host}" added.`
});
} catch (error){
return next(error);
}
});
router.get('/:item(*)', async function(req, res, next){
try{
return res.json({
item: req.params.item,
results: await Model.get(req.params.item)
});
}catch(error){
return next(error);
}
});
router.put('/:item(*)', async function(req, res, next){
try{
req.body.updated_by = req.user.username;
let item = await Model.get(req.params.item);
await item.update.call(item, req.body);
return res.json({
message: `"${req.params.item}" updated.`
});
}catch(error){
return next(error);
}
});
router.delete('/:item(*)', async function(req, res, next){
try{
let item = await Model.get(req.params);
let count = await host.remove.call(item, item);
return res.json({
message: `${req.params.host} deleted`,
});
}catch(error){
return next(error);
}
});
module.exports = router;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment