Skip to content

@mladenmarkov /couchbaseBulkSet.js
Created

Embed URL

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
An example how to workaround node.js couchbase library when saving large number of documents. Use a pool of connections and make sure each connection doesn't get more than around 50 docs at a time.
/**
* An example how to workaround node.js couchbase library when saving large number of documents.
* Use a pool of connections and make sure each connection doesn't get more than around 50 docs at a time.
*
* @author Mladen Markov (mladen.markov@gmail.com)
* @version 1.0 Feb 12, 2013 19:45
*/
var async = require("async");
var couchBase = require("couchbase");
var config = {
"host": "localhost",
"port": 8091,
"username": "Administrator",
"password": "password",
"bucket": "facebook-pages"
};
var buckets;
var current = 0;
async.times(100, function(n, callback) {
couchBase.connect(config, function(err, bucket) {
if (err) {
console.log("Unable to connect to Couchbase bucket [" + config.bucket + "]", err);
process.exit(1);
}
console.log("Connected to Couchbase.");
callback(null, bucket);
});
}, function(err, result) {
if (err) {
console.log("Unable to create Couchbase connection pool.");
process.exit(1);
}
buckets = result;
console.log("Created a Couchbase pool of [" + buckets.length + "] connections");
var jsonDocs = [];
for (var i = 0; i < 4000; i++) {
jsonDocs.push({id:i + ""});
}
console.log("Saving [" + jsonDocs.length + "] docs");
async.map(jsonDocs, function(doc, callback) {
var bucket = getBucket();
bucket.set(doc.id, doc, {}, function(err) {
callback(err);
});
}, function(err, results) {
if (err) {
console.log("Unable to save all entries", err);
process.exit(1);
}
console.log("Saved all entries");
process.exit(0);
});
});
function getBucket() {
if (current >= buckets.length) {
current = 0;
}
return buckets[current++];
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Something went wrong with that request. Please try again.