Skip to content

Instantly share code, notes, and snippets.

@odnarb
Created July 2, 2016 16:27
Show Gist options
  • Save odnarb/2038ab64048ad601f7c005f70aecdaf9 to your computer and use it in GitHub Desktop.
Save odnarb/2038ab64048ad601f7c005f70aecdaf9 to your computer and use it in GitHub Desktop.
var request = require('request');
var _ = require('underscore');
//get data from db
var extractQuery = "select top 1000 * from mytable";
var chunkedPostOptions = {
method: 'POST',
rejectUnauthorized: false, //self-signed ssl
headers: {
'Transfer-Encoding': 'chunked',
'Content-Type': 'multipart/form-data',
'Cookie': "some-cookie here"
},
qs: {
'_csrf': csrf
},
baseUrl: 'https://localhost'
};
var request = require('request');
var postReq = request.defaults(chunkedPostOptions);
var stream = require('stream');
var rs = new stream.Readable();
rs._read = function noop() {}; // redundant?
//open the pipe here
rs.pipe(postReq('/stream_to_file', function(err,res, body){
if( err ){
console.log("Stream Pipe Error!");
console.log(err);
}
}));
//Init db connection & config
var dbConnection = require('tedious').Connection;
var dbConfig = {
server: 'some-far-away-server.net',
userName: 'me',
password:'myp@ss',
options: {
database: 'mydb',
useColumnNames: false //tedious is weird, use this..
}
};
dbConnection = new dbConnection(dbConfig);
var dbRequest = require('tedious').Request;
var dbSetSentRequest = '';
//prep sql exec handlers
var executeStatement = function(cb) {
dbRequest = new dbRequest(extractQuery, function(err, rowCount) {
if (err) {
console.log(err);
cb();
} else {
totalRows = rowCount;
if( totalRows > 0 ) {
if( rowsProcessed == totalRows ) {
//end stream
rs.push(null);
cb();
} //endif
} else {
cb();
} //endif
} //endif
});
dbRequest.on( 'row', function handleRow( cols ) {
rowsProcessed+=1;
//keep pushing content into the stream
//get the row into a clean series of properties like: { id: 123, first_name: 'John', last_name: 'Smith' }
var row = {};
_.each(cols, function(col){
if( col.value === null ) {
row[col.metadata.colName] = null;
} else if( col.metadata.userType !== 80){ //skip unsigned ints (timestamps) formatted like 0x0000000024D61832
row[col.metadata.colName] = col.value.toString();
}
});
var chunkedBody = {
row_number: rowsProcessed,
content_type: 'application/json',
content: JSON.stringify(row)
};
rs.push( JSON.stringify(chunkedBody) + '\r\n' );
});
dbConnection.execSql(dbRequest);
};
//actual connection handler and start execution
var rowsProcessed = 0;
var totalRows = 0;
dbConnection.on('connect', function(err) {
// If no error, then good to go...
if(err){
console.log(err);
} else {
executeStatement(function(){
dbConnection.close();
});
}
});
@odnarb
Copy link
Author

odnarb commented Jul 2, 2016

@Sergey80 @r0hitsharma

Wanted to add you on this, since the original issue was closed.

request/request#401

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment