Created
July 2, 2016 16:27
-
-
Save odnarb/2038ab64048ad601f7c005f70aecdaf9 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var request = require('request'); | |
var _ = require('underscore'); | |
//get data from db | |
var extractQuery = "select top 1000 * from mytable"; | |
var chunkedPostOptions = { | |
method: 'POST', | |
rejectUnauthorized: false, //self-signed ssl | |
headers: { | |
'Transfer-Encoding': 'chunked', | |
'Content-Type': 'multipart/form-data', | |
'Cookie': "some-cookie here" | |
}, | |
qs: { | |
'_csrf': csrf | |
}, | |
baseUrl: 'https://localhost' | |
}; | |
var request = require('request'); | |
var postReq = request.defaults(chunkedPostOptions); | |
var stream = require('stream'); | |
var rs = new stream.Readable(); | |
rs._read = function noop() {}; // redundant? | |
//open the pipe here | |
rs.pipe(postReq('/stream_to_file', function(err,res, body){ | |
if( err ){ | |
console.log("Stream Pipe Error!"); | |
console.log(err); | |
} | |
})); | |
//Init db connection & config | |
var dbConnection = require('tedious').Connection; | |
var dbConfig = { | |
server: 'some-far-away-server.net', | |
userName: 'me', | |
password:'myp@ss', | |
options: { | |
database: 'mydb', | |
useColumnNames: false //tedious is weird, use this.. | |
} | |
}; | |
dbConnection = new dbConnection(dbConfig); | |
var dbRequest = require('tedious').Request; | |
var dbSetSentRequest = ''; | |
//prep sql exec handlers | |
var executeStatement = function(cb) { | |
dbRequest = new dbRequest(extractQuery, function(err, rowCount) { | |
if (err) { | |
console.log(err); | |
cb(); | |
} else { | |
totalRows = rowCount; | |
if( totalRows > 0 ) { | |
if( rowsProcessed == totalRows ) { | |
//end stream | |
rs.push(null); | |
cb(); | |
} //endif | |
} else { | |
cb(); | |
} //endif | |
} //endif | |
}); | |
dbRequest.on( 'row', function handleRow( cols ) { | |
rowsProcessed+=1; | |
//keep pushing content into the stream | |
//get the row into a clean series of properties like: { id: 123, first_name: 'John', last_name: 'Smith' } | |
var row = {}; | |
_.each(cols, function(col){ | |
if( col.value === null ) { | |
row[col.metadata.colName] = null; | |
} else if( col.metadata.userType !== 80){ //skip unsigned ints (timestamps) formatted like 0x0000000024D61832 | |
row[col.metadata.colName] = col.value.toString(); | |
} | |
}); | |
var chunkedBody = { | |
row_number: rowsProcessed, | |
content_type: 'application/json', | |
content: JSON.stringify(row) | |
}; | |
rs.push( JSON.stringify(chunkedBody) + '\r\n' ); | |
}); | |
dbConnection.execSql(dbRequest); | |
}; | |
//actual connection handler and start execution | |
var rowsProcessed = 0; | |
var totalRows = 0; | |
dbConnection.on('connect', function(err) { | |
// If no error, then good to go... | |
if(err){ | |
console.log(err); | |
} else { | |
executeStatement(function(){ | |
dbConnection.close(); | |
}); | |
} | |
}); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
@Sergey80 @r0hitsharma
Wanted to add you on this, since the original issue was closed.
request/request#401