Skip to content

Instantly share code, notes, and snippets.

@matt212
Last active July 12, 2017 10:08
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save matt212/b6449f6354d0f328e8ab80a55afa83ee to your computer and use it in GitHub Desktop.
Save matt212/b6449f6354d0f328e8ab80a55afa83ee to your computer and use it in GitHub Desktop.
index is required !
var elasticsearch = require('elasticsearch');
var ElasticsearchWritableStream = require('elasticsearch-writable-stream');
var connections = require('./config/db');
var client = new elasticsearch.Client({
host: 'localhost:9200',
log: 'trace'
});
client.ping({
requestTimeout: 30000,
}, function (error) {
if (error) {
console.error('elasticsearch cluster is down!');
} else {
console.log('All is well');
}
});
var stream = new ElasticsearchWritableStream(client, {
highWaterMark: 256,
flushTimeout: 500
});
//var connectionString = "postgres://*USERNAME*:*PASSWORD*@*HOST*:*PORT:/*DATABASE*"
var connectionString = "postgres://postgres:abc123@localhost:5432/demodb";
var pg = require('pg')
var QueryStream = require('pg-query-stream')
var JSONStream = require('JSONStream')
/*client.indices.create({
index: 'gov'
},function(err,resp,status) {
if(err) {
console.log(err);
}
else {
console.log("create",resp);
}
});*/
/*deleting an indices*/
/*client.indices.delete({index: 'gov'},function(err,resp,status) {
console.log("delete",resp);
});*/
//pipe 1,000,000 rows to stdout without blowing up your memory usage
pg.connect(connectionString,function(err, client, done) {
if(err) throw err;
var query = new QueryStream('SELECT * FROM srt')
var streams = client.query(query)
//release the client when the stream is finished
streams.on('end', done)
streams.pipe(JSONStream.stringify()).pipe(stream)
})
/*
someInputStream
.pipe(stream)
.on('error', function(error) {
// Handle error
})
.on('finish', function() {
// Clean up Elasticsearch client?
})*/
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment