Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
AWS ES proxy
var AWS = require('aws-sdk');
var http = require('http');
var httpProxy = require('http-proxy');
var express = require('express');
var bodyParser = require('body-parser');
var stream = require('stream');
if (process.argv.length != 3) {
console.error('usage: aws-es-proxy <my-cluster-endpoint>');
process.exit(1);
}
var ENDPOINT = process.argv[2];
var m = ENDPOINT.match(/\.([^.]+)\.es\.amazonaws\.com\.?$/);
if (!m) {
console.error('region cannot be parsed from endpoint address, must end in .<region>.es.amazonaws.com');
process.exit(1);
}
var REGION = m[1];
var TARGET = 'https://' + process.argv[2];
var PORT = 9200;
var BIND_ADDRESS = '127.0.0.1';
var creds;
var chain = new AWS.CredentialProviderChain();
chain.resolve(function (err, resolved) {
if (err) throw err;
else creds = resolved;
});
function getcreds(req, res, next) {
return creds.get(function (err) {
if (err) return next(err);
else return next();
});
}
var proxy = httpProxy.createProxyServer({
target: TARGET,
changeOrigin: true,
secure: true
});
var app = express();
app.use(bodyParser.raw({type: '*/*'}));
app.use(getcreds);
app.use(function (req, res) {
var bufferStream;
if (Buffer.isBuffer(req.body)) {
var bufferStream = new stream.PassThrough();
bufferStream.end(req.body);
}
proxy.web(req, res, {buffer: bufferStream});
});
proxy.on('proxyReq', function (proxyReq, req, res, options) {
var endpoint = new AWS.Endpoint(ENDPOINT);
var request = new AWS.HttpRequest(endpoint);
request.method = proxyReq.method;
request.path = proxyReq.path;
request.region = REGION;
if (Buffer.isBuffer(req.body)) request.body = req.body;
if (!request.headers) request.headers = {};
request.headers['presigned-expires'] = false;
request.headers['Host'] = ENDPOINT;
var signer = new AWS.Signers.V4(request, 'es');
signer.addAuthorization(creds, new Date());
proxyReq.setHeader('Host', request.headers['Host']);
proxyReq.setHeader('X-Amz-Date', request.headers['X-Amz-Date']);
proxyReq.setHeader('Authorization', request.headers['Authorization']);
if (request.headers['x-amz-security-token']) proxyReq.setHeader('x-amz-security-token', request.headers['x-amz-security-token']);
});
http.createServer(app).listen(PORT, BIND_ADDRESS);
console.log('listening at ' + BIND_ADDRESS + ':' + PORT);
@lerrigatto

This comment has been minimized.

Show comment Hide comment
@lerrigatto

lerrigatto Nov 25, 2015

Hi! This is great!

Hi! This is great!

@mooreds

This comment has been minimized.

Show comment Hide comment
@mooreds

mooreds Jan 12, 2016

@nakedible-p, did you end up deciding to make this a npm module? Just curious.

Thanks for providing this!

mooreds commented Jan 12, 2016

@nakedible-p, did you end up deciding to make this a npm module? Just curious.

Thanks for providing this!

@lfrancois

This comment has been minimized.

Show comment Hide comment
@lfrancois

lfrancois Jan 27, 2016

Hi @nakedible-p!

First thanks for your work on this proxy. I try to use it with my fresh AWS ES cluster, but I get this error. Any idea why?

root@kibana ~ # node es-aws-proxy.js 
usage: aws-es-proxy <my-cluster-endpoint>
root@kibana ~ #

root@kibana ~ # node es-aws-proxy.js search-mycluster-XXXXXXXXXXX.us-west-2.es.amazonaws.com
listening at 127.0.0.1:9200

/root/es-aws-proxy.js:26
    if (err) throw err;
                   ^
SyntaxError: Unexpected token <
    at Object.parse (native)
    at /root/node_modules/aws-sdk/lib/metadata_service.js:115:38
    at IncomingMessage.<anonymous> (/root/node_modules/aws-sdk/lib/metadata_service.js:74:45)
    at IncomingMessage.emit (events.js:117:20)
    at _stream_readable.js:944:16
    at process._tickDomainCallback (node.js:492:13)
root@kibana ~ # 

Hi @nakedible-p!

First thanks for your work on this proxy. I try to use it with my fresh AWS ES cluster, but I get this error. Any idea why?

root@kibana ~ # node es-aws-proxy.js 
usage: aws-es-proxy <my-cluster-endpoint>
root@kibana ~ #

root@kibana ~ # node es-aws-proxy.js search-mycluster-XXXXXXXXXXX.us-west-2.es.amazonaws.com
listening at 127.0.0.1:9200

/root/es-aws-proxy.js:26
    if (err) throw err;
                   ^
SyntaxError: Unexpected token <
    at Object.parse (native)
    at /root/node_modules/aws-sdk/lib/metadata_service.js:115:38
    at IncomingMessage.<anonymous> (/root/node_modules/aws-sdk/lib/metadata_service.js:74:45)
    at IncomingMessage.emit (events.js:117:20)
    at _stream_readable.js:944:16
    at process._tickDomainCallback (node.js:492:13)
root@kibana ~ # 
@lfrancois

This comment has been minimized.

Show comment Hide comment
@lfrancois

lfrancois Jan 27, 2016

I forgot to set correctly my AWS IAM information, now it works better!

root@kibana ~ # export AWS_ACCESS_KEY_ID=XXXXXXXXXX
root@kibana ~ # export AWS_SECRET_ACCESS_KEY=YYYYYYY

I forgot to set correctly my AWS IAM information, now it works better!

root@kibana ~ # export AWS_ACCESS_KEY_ID=XXXXXXXXXX
root@kibana ~ # export AWS_SECRET_ACCESS_KEY=YYYYYYY
@vevo-john-delivuk

This comment has been minimized.

Show comment Hide comment
@vevo-john-delivuk

vevo-john-delivuk Feb 11, 2016

@nakedible-p Would you be interested in making it an npm? I'd be happy to setup the project and give you full access. I just want to get this out there for others to use as easily as possible. I've heard from AWS there is a solution coming in the near future to alleviate the need for this, but as of right now this is by far the best solution I've found. Kudos sir!

@nakedible-p Would you be interested in making it an npm? I'd be happy to setup the project and give you full access. I just want to get this out there for others to use as easily as possible. I've heard from AWS there is a solution coming in the near future to alleviate the need for this, but as of right now this is by far the best solution I've found. Kudos sir!

@chiefy

This comment has been minimized.

Show comment Hide comment
@chiefy

chiefy Mar 12, 2016

I wish I had this two months ago, thanks so much! AWS should put it in their docs!

chiefy commented Mar 12, 2016

I wish I had this two months ago, thanks so much! AWS should put it in their docs!

@santthosh

This comment has been minimized.

Show comment Hide comment
@santthosh

santthosh Mar 30, 2016

Added an npm module for this aws-es-kibana. Thanks @nakedible-p

Added an npm module for this aws-es-kibana. Thanks @nakedible-p

@satheessh

This comment has been minimized.

Show comment Hide comment
@satheessh

satheessh Apr 20, 2016

getting below error fluentd + proxy. however simple curl (localhost:9200) works fine.

error_class="Elasticsearch::Transport::Transport::Errors::Forbidden" error="[403] {"message":"The request signature we calculated does not match the signature you provided. Check your AWS Secret Access Key and signing method. Consult the service documentation for details.\n\nThe Canonical String for this request should have been\n'POST\n/_bulk\n\nhost:search-elastic-search-fx2ecksqldt5uoljqaawt655fa.us-east-1.es.amazonaws.com\nx-amz-date:20160420T093625Z\n\nhost;x-amz-date\n98a6e169f9136854c5dd591dcd23a606cd70311ca643bcde483c87dbad18ff2f'\n\nThe String-to-Sign should have been\n'AWS4-HMAC-SHA256\n20160420T093625Z\n20160420/us-east-1/es/aws4_request\n0cb3e76bfed7f8b25715e4ab35cead39054dface473512275c6f832d2ea11b8a'\n"}"

getting below error fluentd + proxy. however simple curl (localhost:9200) works fine.

error_class="Elasticsearch::Transport::Transport::Errors::Forbidden" error="[403] {"message":"The request signature we calculated does not match the signature you provided. Check your AWS Secret Access Key and signing method. Consult the service documentation for details.\n\nThe Canonical String for this request should have been\n'POST\n/_bulk\n\nhost:search-elastic-search-fx2ecksqldt5uoljqaawt655fa.us-east-1.es.amazonaws.com\nx-amz-date:20160420T093625Z\n\nhost;x-amz-date\n98a6e169f9136854c5dd591dcd23a606cd70311ca643bcde483c87dbad18ff2f'\n\nThe String-to-Sign should have been\n'AWS4-HMAC-SHA256\n20160420T093625Z\n20160420/us-east-1/es/aws4_request\n0cb3e76bfed7f8b25715e4ab35cead39054dface473512275c6f832d2ea11b8a'\n"}"

@jindov

This comment has been minimized.

Show comment Hide comment
@jindov

jindov Aug 21, 2017

+1 you save my day
I use supervisor to control this, configure with AWS variables
Thanks

jindov commented Aug 21, 2017

+1 you save my day
I use supervisor to control this, configure with AWS variables
Thanks

@nicokruger

This comment has been minimized.

Show comment Hide comment
@nicokruger

nicokruger Sep 15, 2017

+1, this works brilliantly!

+1, this works brilliantly!

@joonaskaskisola

This comment has been minimized.

Show comment Hide comment
@joonaskaskisola

joonaskaskisola Nov 29, 2017

Excellent, thank you!

Excellent, thank you!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment