Create a gist now

Instantly share code, notes, and snippets.

@renedx /couchPool
Last active Dec 13, 2015

What would you like to do?
Simple Couchbase pooling, still searching for a method to close existing connections.
var Config = require( '../config' )
, couchbase = require( 'couchbase' )
, couchPool = {
base: {
'hosts': Config.couchbase.hosts,
'port': 8091,
'username': Config.couchbase.username,
'password': Config.couchbase.password
},
minimalConnections: 5,
buckets: [ 'default' ],
connections: {},
setup: function(cb) {
var connection = 0;
return couchPool.buckets.forEach(function(bucketName){
var i;
for( i = 0; i < couchPool.minimalConnections; i++ ) {
couchPool.addConnection(bucketName, function(){
connection++;
if( connection == ( couchPool.buckets.length * couchPool.minimalConnections ) ) {
cb(connection)
}
}, false)
}
return;
})
},
addConnection: function(bucketName, cb, used) {
console.log( 'addConnection: ' + bucketName )
try {
couchbase.connect({
'hosts': couchPool.base.hosts,
'port': couchPool.base.port,
'username': couchPool.base.username,
'password': couchPool.base.password,
'bucket': bucketName,
'debug': false
}, function(err, bucket){
if( err )
return console.log(err);
if( !( bucketName in couchPool.connections ) )
couchPool.connections[( bucketName )] = [];
var k = couchPool.connections[( bucketName )].push({
active: used,
bucket: bucket,
died: false
})
return cb(bucket, (k-1))
})
}
catch( e ) {
console.log( '[couchpool] error ' + e)
// Retry in next tick
process.nextTick(function(){
couchPool.addConnection(bucketName, cb, used)
})
return;
}
},
removeConnection: function(bucketName, k){
if( ( bucketName in couchPool.connections )
&& couchPool.connections[( bucketName )][( k )] ) {
console.log( '[couchpool] free up connection' )
couchPool.connections[( bucketName )][( k )].active = false;
}
},
free: function(bucketName, k) {
console.log( '[couchpool] Free up connection ' + bucketName )
couchPool.removeConnection(bucketName, k)
if( !( bucketName in couchPool.connections ) )
return console.log( '[Couchpool] Bucket ' + bucketName + ' doesn\'t exist' );
var activeConnections = 0;
couchPool.connections[( bucketName )].every(function(connection, k){
if( connection.active === true
&& connection.died === false ) {
activeConnections++;
if( activeConnections > couchPool.minimalConnections ) {
// connection.bucket.close() -- currently no close method supported
}
}
return true;
})
},
get: function(bucketName, cb) {
if( !( bucketName in couchPool.connections ) )
return console.log( '[Couchpool] Bucket ' + bucketName + ' doesn\'t exist' );
var found = false, closeHandler = function(connection, bucketName, k){
return cb(connection, function(){
couchPool.free(bucketName, k)
})
};
couchPool.connections[( bucketName )].every(function(connection, k){
if( connection.active === false
&& connection.died === false ) {
// Set used
couchPool.connections[( bucketName )][( k )].active = true;
closeHandler(connection.bucket, bucketName, k)
found = true;
return false;
}
return true;
})
// Found something?
if( !found ) {
console.log( 'no free connection, request a new one' )
return couchPool.addConnection( bucketName, function(c, k){
closeHandler(c, bucketName, k)
}, true )
}
else
return;
}
};
couchPool.setup(function(c){
console.log( 'Buckets: ' + c )
// Run crazy code
var i = 0, ids = [], runDate = Math.floor(new Date().getTime()/1000);
for( i = 0; i <= 300; i++ )
ids.push(i)
console.log(ids)
ids.every(function(i){
console.log( 'request bucket' )
couchPool.get( 'default', function(bucket, done){
console.log( 'add item: ' + i )
bucket.add( 'invoices::test::' + runDate + '::#' + i, { 'test': 'test', 'id': i }, function(err, result){
if( err )
console.log(err)
console.log(result)
return done()
})
})
return true;
})
})

Well, I didn't need to close the connections - I keep all open as they are used all the time. The server is quite loaded at all times.

I think I'll abandon this approach, though. There's no reliable way to ensure that you will not accidentally overload one connection. And if you do, there's no going back - node just hangs and consumes CPU.

Owner

renedx commented Feb 14, 2013

Haha, I'll keep with the modified cradle for now.
Couchbase lib doesn't take async calls in account and that's a shame.

By the way, the Java SDK seems much more stable and faster.
I implemented a quick and dirty REST proxy as a Java app -- 200 lines of code using Jetty -- and now I can feed it as many docs as I want and it saves them 4 times faster.

The drawback is that you need to run another process to proxy all Couchbase communication. I need only gets() and cas() though

The Java SDK lacks multiple get, which is sad

Owner

renedx commented Feb 14, 2013

Mmm, that doesn't sound like a bad idea. Would you like to share it with us? :)

@mladenmarkov That got our attention, as we're seriously struggling.. care to share your Java code? We're not too familiar with Java.

Owner

renedx commented Feb 14, 2013

Nvm. We now use node-memcache with baseview :)

@DeviaVir I'll create a GitHub project tomorrow
@renedx I already tried node-memcache earlier today, but it wouldn't connect to my Couchbase 2.0 Server

Owner

renedx commented Feb 15, 2013

@mladenmarkov It didn't connect here first because of the firewall blocking it. Now it just works and fast :)
The port given should be the default memcache port. You can define a memcache port on each bucket in the Couchbase Server. We also run 2.0.

@DeviaVir, here's the Couchbase Java Proxy, if you're still interested. I'm using it in production and it works great. It's even faster than node-memcache and memcached. That is with CAS, which I need to use.

https://github.com/mladenmarkov/couchbase-proxy

@trondn told me he's working on the problem, so hopefully it should be fixed soon. I'd rather use a node-only solution than have to manage another process for proxying Couchbase calls.
http://www.couchbase.com/issues/browse/JSCBC-14

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment