Last active
July 12, 2017 10:08
-
-
Save matt212/b6449f6354d0f328e8ab80a55afa83ee to your computer and use it in GitHub Desktop.
index is required !
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var elasticsearch = require('elasticsearch'); | |
var ElasticsearchWritableStream = require('elasticsearch-writable-stream'); | |
var connections = require('./config/db'); | |
var client = new elasticsearch.Client({ | |
host: 'localhost:9200', | |
log: 'trace' | |
}); | |
client.ping({ | |
requestTimeout: 30000, | |
}, function (error) { | |
if (error) { | |
console.error('elasticsearch cluster is down!'); | |
} else { | |
console.log('All is well'); | |
} | |
}); | |
var stream = new ElasticsearchWritableStream(client, { | |
highWaterMark: 256, | |
flushTimeout: 500 | |
}); | |
//var connectionString = "postgres://*USERNAME*:*PASSWORD*@*HOST*:*PORT:/*DATABASE*" | |
var connectionString = "postgres://postgres:abc123@localhost:5432/demodb"; | |
var pg = require('pg') | |
var QueryStream = require('pg-query-stream') | |
var JSONStream = require('JSONStream') | |
/*client.indices.create({ | |
index: 'gov' | |
},function(err,resp,status) { | |
if(err) { | |
console.log(err); | |
} | |
else { | |
console.log("create",resp); | |
} | |
});*/ | |
/*deleting an indices*/ | |
/*client.indices.delete({index: 'gov'},function(err,resp,status) { | |
console.log("delete",resp); | |
});*/ | |
//pipe 1,000,000 rows to stdout without blowing up your memory usage | |
pg.connect(connectionString,function(err, client, done) { | |
if(err) throw err; | |
var query = new QueryStream('SELECT * FROM srt') | |
var streams = client.query(query) | |
//release the client when the stream is finished | |
streams.on('end', done) | |
streams.pipe(JSONStream.stringify()).pipe(stream) | |
}) | |
/* | |
someInputStream | |
.pipe(stream) | |
.on('error', function(error) { | |
// Handle error | |
}) | |
.on('finish', function() { | |
// Clean up Elasticsearch client? | |
})*/ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment