Last active
December 14, 2015 23:49
-
-
Save soldair/5168249 to your computer and use it in GitHub Desktop.
how to run a copy command with data from any node stream into to a vertica database
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var through = require('through'); | |
// pure js vertica driver. im not into the coffee script but this is so useful even i domnt hold it against them | |
var vertica = require('vertica'); | |
var stream = through(); | |
// | |
// pipe some bar delimited rows of data to your stream. | |
// until you call stream.end() the data will not commit. | |
// because you can use logical streams rather than "real" streams.. | |
// .. you can break a job up simply by ending the stream and starting another copy command .. | |
// .. with a new stream. | |
// this should help reduce the risk of loosing uncommitted data on a crash ;) | |
// | |
var sql = "COPY table(fields) FROM STDIN DELIMITER '|';"; | |
var con = vertica.connect(credentials); | |
con.copy(sql,function(transfer, success, fail){ | |
stream.on('data',function(data){ | |
transfer(data); | |
}); | |
stream.on('end',function(data){ | |
if(data) { | |
transfer(data); | |
} | |
success(); | |
}); | |
stream.on('error',function(err){ | |
fail(); | |
}); | |
},function(err,data){ | |
// done. | |
}); | |
var i = 0,interval; | |
interval = setInterval(function(){ | |
++i | |
if(i > 10) { | |
stream.end(); | |
clearInterval(interval); | |
} | |
stream.write(new Buffer("hi|i|am|values\n")); | |
},1000); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment