Oloo Reports - Pipe large file - Async builds - zip password and email.

1olooReport.js

/**
 * Report Creation - OLOO Method
 */
var BPromise = require('bluebird');
var _ = require('lodash');
var fs = BPromise.promisifyAll(require("fs")); //This is most convenient way if it works for you
var path = require('path');
var moment = require('moment-timezone');

var Account = require('../../src/models/Account');
var AccountCard = require('../../src/models/AccountCard');
var csv_util = require('../../src/utils/csv_util');
var zip_util = require('../../src/utils/zip_util');
var sendgridUtils = require('../../src/cards_email/sendgrid.utils.js')();


module.exports = {
        init: function(initObjIn) {
            var initObj = _.extend({}, {
                report_name: null,
                email_to: null,
            }, initObjIn);

            this.report_name = initObj.report_name || 'report';
            this.email_to = initObj.email_to;
            this.email_from = "x@t.com";
            this.path_to_report_files = 'src/report_data/';


            this.create_csv_and_save_file = BPromise.method(function(report_array) {

                if(_.isEmpty(report_array)){
                    throw('Report is empty, nothing to convert to csv');
                }

                this.csvPath = this.path_to_report_files + this.report_name + ".csv";
                //like "src/report_data/filetest.csv"

                return csv_util.convertToCsv(report_array)
                    .bind(this)
                    .then(function(csv){
                        // todo, verify csv was converted
                        console.log('Array was converted to CSV');
                        return csv;
                    })

                    // Write csv to file
                    .then(function(csv){
                        return fs.writeFileAsync(this.csvPath, csv, 'utf-8')
                            .bind(this)
                            .then(function(ret){
                                // TODO, VERIFY FILE
                                return this.csvPath;
                            });
                    })
                    .catch(function(err) {
                        if(err){
                            console.log('INFO: ' + new Date() + ': create_csv_and_save_file failed: ' + err);
                        }
                        return false;
                    });
            });


            this.zip_csv_file_paths = BPromise.method(function(csvFileArray) {

                if(_.isEmpty(csvFileArray)){
                    throw('csvFileArray is empty, nothing to zip');
                }

                this.zip_file_name = this.report_name + '.zip';
                this.zip_file_path = this.path_to_report_files + this.zip_file_name;
                this.zip_password = getZipPassword();

                // var csvFileArray = [
                //     'src/report_data/naco_report.csv',
                //     'src/report_data/rxsp_report.csv'
                // ];

                return zip_util.zip_it_w_password(
                        csvFileArray,
                        this.zip_password,
                        this.zip_file_path,
                        {}
                    )
                    .bind(this)
                    .then(function(zipResponse){
                        console.log('zip_util.zip_it_w_password complete ');
                        return this.zip_file_path;
                    })
                    .catch(function(err) {
                        if(err){
                            console.log('INFO: ' + new Date() + ': zip_csv_file_paths failed: ' + err);
                        }
                        return false;
                    });
            });


            this.email_zip_attachment = BPromise.method(function(zipFilePath) {

                if (_.isEmpty(zipFilePath)) {
                    throw('zipFilePath is empty, nothing to zip');
                }

                return fs.readFileAsync(zipFilePath)
                    .bind(this)
                    .then(function (content) {
                        var bas64 = new Buffer(content).toString('base64');
                        return bas64;
                    })

                    // Send email with zip attachment
                    .then(function (attachData) {
                        console.log('We have attachData now send email with report zip attached');

                        var options = {
                            "email_to": "f@d.com",
                            "email_from": "c@f.com",
                            "content_body": this.report_name,
                            "subject": this.report_name,
                            "category": ["report"],
                            "attachments": [
                                {
                                    "content": attachData,
                                    "type": "application/zip",
                                    "filename": this.zip_file_name,
                                    "disposition": "attachment"
                                }
                            ]
                        };

                        return sendgridUtils.sendTemplate(options)
                            .then(function (res) {
                                console.log('Send email code: ' + res.statusCode);
                                return res.statusCode;
                            });
                    })
                    .catch(function (err) {
                        if (err) {
                            console.log('INFO: ' + new Date() + ': REPORT failed: ' + err);
                        }
                        return false;
                    });
            });
        },

        create_csv_by_account_type_n_cardalpha: BPromise.method(function(account_type_in, card_alpha_in) {
            console.log('/n create_csv_by_account_type_n_cardalpha');
            var account_type   = account_type_in || null;
            var card_alpha   = card_alpha_in || null;

            console.log( ' Report will run on account type ' + account_type);
            if(!_.isNull(card_alpha_in)){
                console.log( ' Report will run on Card Alpha ' + card_alpha);
            }
            console.log( this.report_name + ' will be emailed to ' + this.email_to);


            return AccountCard.fetchAll({
                    withRelated: ['account']
                })
                .bind(this)
                .then(function(allCards){
                    console.log('-- We have all cards with account info, next filter: ' + allCards.length);
                    return allCards.models
                })
                // Filter on account type
                .filter(function(card) {
                    return card.relations.account.attributes.account_type_id === account_type;
                })
                // Filter on card alpha
                .filter(function(card) {
                    if( _.isNull(card_alpha) ){
                        return true;
                    }
                    else if( !_.isNull(card_alpha) && _.startsWith(card.attributes.id, card_alpha) ){
                        return true;
                    }
                })
                // add account type to data
                .reduce(function(newArray, card){
                    if( !_.isUndefined(card.attributes) ){
                        // var newCard = _.omitBy(card, _.isNil);
                        card.attributes.account_type_id = card.relations.account.attributes.account_type_id;
                        newArray.push(card.attributes);
                        return newArray;
                    }
                },[])
                // check data then send along
                .then(function(newCards){
                    if( _.isEmpty(newCards) ) {
                        throw "No records found for this report";
                    } else {
                        console.log('Convert to csv and return it');
                        return this.create_csv_and_save_file(newCards)
                            .then(function(cdfPath){
                                console.log('CSV was created');
                                return cdfPath;
                        });

                    }
                });
        }),

        create_csv_by_account_id_n_cardalpha: BPromise.method(function(account_id_in, card_alpha_in) {
            var account_id   = account_id_in || null;
            var card_alpha   = card_alpha_in || null;

            console.log( ' Report will run on account ID ' + account_id);
            if(!_.isNull(card_alpha_in)){
                console.log( ' Report will run on Card Alpha ' + card_alpha);
            }
            console.log( this.report_name + ' will be emailed to ' + this.email_to);

            return AccountCard.getCardsByAccount(account_id)
                .bind(this)
                .then(function(allCards){
                    console.log('-- We have all cards with account info: ' + allCards.length);
                    if(_.isEmpty(allCards)){
                        throw "No records found for account id " + account_id;
                    }
                    if(!_.isNull(card_alpha)){
                        console.log()
                    }
                    return allCards
                })
                // Filter on card alpha
                .filter(function(card) {
                    if( _.isNull(card_alpha) ){
                        return true;
                    }
                    else if( !_.isNull(card_alpha) && _.startsWith(card.attributes.id, card_alpha) ){
                        return true;
                    }
                })
                // Jut send attributes, add account type to data
                .reduce(function(newArray, card){
                    if( !_.isUndefined(card.attributes) ){
                        // var newCard = _.omitBy(card, _.isNil);
                        card.attributes.account_type_id = card.relations.account.attributes.account_type_id;
                        newArray.push(card.attributes);
                        return newArray;
                    }
                },[])
                // check data then send it along
                .then(function(newCards){
                    if(!_.isEmpty(newCards)){

                        console.log('Convert to csv and return');
                        return this.create_csv_and_save_file(newCards)
                            .then(function(cdfPath){
                                console.log('CSV was created');
                                return cdfPath;
                        });
                    }
                });
        }),
};



function getZipPassword(){
    if (_.isEmpty(process.env.REPORT_PASSWORD) ) {
        throw new Error('process.env.REPORT_PASSWORD is missing.')
    } else if (typeof process.env.REPORT_PASSWORD !== 'string') {
        throw new TypeError('zip_util: process.env.REPORT_PASSWORD must be a string')
    }
    else {
        return process.env.REPORT_PASSWORD;
    }
}

2used_cards_sync.js

var BPromise = require('bluebird');
var _ = require('lodash');

var fs = require('fs');
BPromise.promisifyAll(fs);

var stream = require('stream');
var es = require('event-stream');

var parse = require('csv-parse');

var Account = require('../../src/models/Account');
var AccountCard = require('../../src/models/AccountCard');
var Report = require('./Report');
var ftpUtil = require('../../src/utils/ftp_util');
var zipUtil = require('../../src/utils/zip_util');

// CONFIG
var email_to = "f@g.com";
var report_name = "d@d.com";
var checkBufferInterval = 1000 * 15;

// FTP CONFIG
var username;
var password;
try {
    username = process.env.FTPUSER;
    password = process.env.FTPPASS;
} catch (e) {
    console.error('Could not get ftp credentials');
}
var ftpConfig = {
    host: "webtransport.advancepcsrx.com",
    username: username,
    password: password,
    // debug: true
    // port: port,
};

var fileInfo = {
    fileDirectory_get: '/discount_webcards/outgoing/',
    fileName_get: 'active_users.small.zip',
    fileDirectory_put: 'src/report_data/',
    fileName_put: 'active_users.zip',
};

var processRow = function(row){
    return new BPromise(function (resolve, reject) {

        // Fixing up - No consistent delimiter, makeing the best of it.
        var parts = _.split(row, "   ");
        var compactParts = _.compact(parts);
        var data = _.map(compactParts, function(string){
            return _.trim(string);
        });

        var usedCardPost = {
            id:  data["1"],
            first_name:  data["3"]
        };

        return AccountCard.searchCardsFirstNameId(usedCardPost)
            .then(function(card) {

                if(_.isNil(card)){
                    resolve(false);
                }

                // console.log('update: ', card.id);
                return AccountCard.usedCard(usedCardPost)
                    .bind(this)
                    .then(function(ret){
                        console.log('Card found and updated: ', usedCardPost);

                        card.attributes.used = true;
                        resolve(card.attributes);
                    })
                    .catch(function(err) {
                        resolve(false);
                    });
            });
    });
};


var usedCardsReport = function() {
    var d = new Date();
    console.log(' \n usedCards REPORT IS STARTING-------------------- '+ d.toLocaleTimeString());

    var cardsUpdated = [];

    return new BPromise(function (resolve, reject) {

        var cardsUpdated = [];
        var linesQued = 0;
        var linesProcessed = 0;
        var bufferActive;
        var bufferLimit = 3000;
        var docLoaded = false;

        return ftpUtil.sftpDownload(ftpConfig, fileInfo)
            .bind(this)
            .then(function(file){
                console.log('we have the zip file, now expand and save');

                var unzipConfig = {
                    sourceZipFilePath: fileInfo.fileDirectory_put,
                    sourceZipFileName: fileInfo.fileName_put,
                    expectFileName: 'active_users.txt',
                };

                var unzippedFileName = fileInfo.fileDirectory_put + unzipConfig.expectFileName;
                return zipUtil.unzip(unzipConfig)
                    .then(function(fileIsUnzipped){
                        console.log('File is Unzipped: ', fileIsUnzipped);
                        return(unzippedFileName);
                    });
            })
            .then(function(file){
                console.log('Got the file to process - via ftp');

                var usedCardsReportObj = Object.create(Report);
                usedCardsReportObj.init({
                    report_name: "usedCards_report",
                    email_to: email_to
                });

                setInterval(checkBuffer, checkBufferInterval);
                function checkBuffer() {
                    bufferActive = linesQued - linesProcessed;
                    if( bufferActive <= bufferLimit ){
                        var d = new Date();
                        console.log('Processed: ' + linesProcessed + ', batch = ' + bufferActive + ' : ' + d.toLocaleTimeString() );

                        if(docLoaded === true && bufferActive == 0 && linesQued === linesProcessed ){
                            console.log("Finished loading doc and all lines are processed");
                            if(_.isEmpty(cardsUpdated)){
                                return reject("No used cards to update");
                            }
                            // Now create and send report of cards that were updated.
                            return usedCardsReportObj.create_csv_and_save_file(cardsUpdated)
                                .then(function(cdfPath){

                                    if(cdfPath === false){
                                        throw "No used cards to update";
                                    }
                                    console.log('CSV was created');
                                    return vdfPath;
                                })
                                .then(function(csvPath){
                                    console.log('Finished creating ', csvPath);
                                    return usedCardsReportObj.zip_csv_file_paths([csvPath]).then(function(zipPath){
                                        return zipPath;
                                    });
                                })

                                .then(function(zipPath){
                                    console.log('Zip created ', zipPath);
                                    return usedCardsReportObj.email_zip_attachment(zipPath).then(function(emailResponse){
                                        console.log('Zip emailed ', emailResponse);
                                        resolve(emailResponse);
                                    });
                                })

                                .catch(function(err) {
                                    if(err){ console.log('INFO: ' + new Date() + ': REPORT ended: ' + err); }
                                    reject(err);
                                    // return false;
                                });

                        } else {
                            s.resume();
                        }
                    }
                }

                var testFile = 'src/report_data/ant928.processed_on_20161202180248.txt';
                var s = fs.createReadStream(file)
                    .pipe(es.split())
                    .pipe(es.mapSync(function(line){
                            linesQued += 1;

                            bufferActive = linesQued - linesProcessed;
                            if( bufferActive > bufferLimit ){
                                s.pause();
                            }

                            return processRow(line)
                                .bind(this)
                                .then(function(card){
                                    linesProcessed += 1;
                                    if(!_.isEmpty(card)){
                                        cardsUpdated.push(card);
                                    }
                                });
                            })
                            .on('error', function(err){
                                console.log('Error while reading file.', err);
                                return reject('Error while reading file.');
                            })
                            .on('close', function (ret) {
                                console.log('Close');
                            })
                            .on('end', function(){
                                console.log('Queued up all lines.');
                                docLoaded = true;
                            })
                    )
            })
            .catch(function(err){
                console.log(err);
            });
    });
};



module.exports = usedCardsReport;


// TSV PARSING - FUTURE...

// var convertFileToCsv = function(){
//     return new BPromise(function (resolve, reject) {
//
//         console.log('fetchFtpFile');
//         var contents = fs.readFileSync(testFile, 'utf8');
//         if (!contents) {
//             feedMessage('Could not read the tsvfile file');
//         }
//
//             // delimiter: '\t',
//
//         var parseOptions = {
//             auto_parse: true, // Ensures that numeric values remain numeric
//             columns: false,
//             comment: '#',
//             delimiter: '\t',
//             quote: '',
//             relax: true,
//             rowDelimiter: '\n', // This is an issue, I had to set the \n here as 'auto' wasn't working, nor was 'windows'.  Maybe look at auto-detecting line endings?
//             skip_empty_lines: true,
//             trim: true
//         };
//
//         parse(contents, parseOptions, function(err, output){
//             if (err){
//                 console.error('csv-parse error', err);
//             }
//             // console.dir('csvoutput', output);
//             resolve(output);
//         });
//     });
// };

2webcards_report.js

var BPromise = require('bluebird');
var _ = require('lodash');

var Report = require('./Report');

var webcardReport = function() {
    console.log(' \n WEBCARD REPORT IS STARTING--------------------');


    var csvFiles = [];

    var webcardReportObj = Object.create(Report);

    webcardReportObj.init({
        report_name: "webcard_report",
        email_to: "cardtrac@gmail.com"
    });


    var p1 = new BPromise(function (f, r) {
        console.log(' \n Co-Branded REPORT IS STARTING--------------------');

        var cobrandedObj = Object.create(Report);
        var cobrandedAccountType = 2;

        cobrandedObj.init({
            report_name: "cobranded_report",
            email_to: null
        });

        return cobrandedObj.create_csv_by_account_type_n_cardalpha(cobrandedAccountType)
            .then(function(csvPath){
                console.log('Finished creating ', csvPath);
                csvFiles.push(csvPath);
                f(csvPath);
            })
            .catch(function(err) {
                r(err);
            });
    });

    var p2 = new BPromise(function (f, r) {
        console.log(' \n isave REPORT IS STARTING--------------------');

        var isaveReportObj = Object.create(Report);
        var isaveAccountId = 5821;

        isaveReportObj.init({
            report_name: "isave_report",
            email_to: "cardtrac@gmail.com"
        });

        return isaveReportObj.create_csv_by_account_id_n_cardalpha(isaveAccountId)
            .then(function(csvPath){
                console.log('Finished creating ', csvPath);
                csvFiles.push(csvPath);
                f(csvPath);
            })
            .catch(function(err) {
                r(err);
            });
    });


    var p3 = new BPromise(function (f, r) {
        console.log(' \n NACo FREE REPORT IS STARTING--------------------');

        var nacoFreeReportObj = Object.create(Report);
        var nacoAccountType = 4;
        var nacoFreeCardAlpha = 'W1B';

        nacoFreeReportObj.init({
            report_name: "naco_free_report",
            email_to: null
        });

        return nacoFreeReportObj.create_csv_by_account_type_n_cardalpha(nacoAccountType, nacoFreeCardAlpha)
            .then(function (csvPath) {
                console.log('Finished creating ', csvPath);
                csvFiles.push(csvPath);
                f(csvPath);
            })
            .catch(function(err) {
                r(err);
            });
    });


    var p4 = new BPromise(function (f, r) {
        console.log(' \n NACo PAID REPORT IS STARTING--------------------');

        var nacoPaidReportObj = Object.create(Report);
        var nacoAccountType = 4;
        var nacoPaidCardAlpha = 'D2X';

        nacoPaidReportObj.init({
            report_name: "naco_paid_report",
            email_to: "cardtrac@gmail.com"
        });

        return nacoPaidReportObj.create_csv_by_account_type_n_cardalpha(nacoAccountType, nacoPaidCardAlpha)
            .then(function(csvPath){
                console.log('Finished creating ', csvPath);
                csvFiles.push(csvPath);
                f(csvPath);
            })
            .catch(function(err) {
                r(err);
            });
    });

    var p5 = new BPromise(function (f, r) {
        console.log(' \n NLC FREE REPORT IS STARTING--------------------');

        var nlcFreeReportObj = Object.create(Report);
        var nlcAccountType = 5;
        var nlcFreeCardAlpha = 'W1B';

        nlcFreeReportObj.init({
            report_name: "nlc_free_report",
            email_to: null
        });

        return nlcFreeReportObj.create_csv_by_account_type_n_cardalpha(nlcAccountType, nlcFreeCardAlpha)
            .then(function (csvPath) {
                console.log('Finished creating ', csvPath);
                csvFiles.push(csvPath);
                f(csvPath);
            })
            .catch(function(err) {
                r(err);
            });
    });


    var p6 = new BPromise(function (f, r) {
        console.log(' \n NLC PAID REPORT IS STARTING--------------------');

        var nlcPaidReportObj = Object.create(Report);
        var nlcAccountType = 5;
        var nlcPaidCardAlpha = 'D2X';

        nlcPaidReportObj.init({
            report_name: "nlc_paid_report",
            email_to: "cardtrac@gmail.com"
        });

        return nlcPaidReportObj.create_csv_by_account_type_n_cardalpha(nlcAccountType, nlcPaidCardAlpha)
            .then(function(csvPath){
                console.log('Finished creating ', csvPath);
                csvFiles.push(csvPath);
                f(csvPath);
            })
            .catch(function(err) {
                r(err);
            });
    });

    var p7 = new BPromise(function (f, r) {
        console.log(' \n Private Label REPORT IS STARTING--------------------');

        var privatelabelObj = Object.create(Report);

        privatelabelObj.init({
            report_name: "privatelabel_report",
            email_to: "cardtrac@gmail.com"
        });

        return privatelabelObj.create_csv_by_account_type_n_cardalpha(3)
            .then(function(csvPath){
                console.log('Finished creating ', csvPath);
                csvFiles.push(csvPath);
                f(csvPath);
            })
            .catch(function(err) {
                r(err);
            });
    });

    var p8 = new BPromise(function (f, r) {
        console.log(' \n rxspmobile REPORT IS STARTING--------------------');

        var rxspmobilereportObj = Object.create(Report);
        var rxspmobileAccountId = 5820;

        rxspmobilereportObj.init({
            report_name: "rxspmobile_report",
            email_to: "cardtrac@gmail.com"
        });

        return rxspmobilereportObj.create_csv_by_account_id_n_cardalpha(rxspmobileAccountId)
            .then(function(csvPath){
                console.log('Finished creating ', csvPath);
                csvFiles.push(csvPath);
                f(csvPath);
            })
            .catch(function(err) {
                r(err);
            });
    });

    var p9 = new BPromise(function (f, r) {
        console.log(' \n rxsp REPORT IS STARTING--------------------');

        var rxspreportObj = Object.create(Report);
        var rxspAccountId = 5819;

        rxspreportObj.init({
            report_name: "rxsp_report",
            email_to: "cardtrac@gmail.com"
        });

        return rxspreportObj.create_csv_by_account_id_n_cardalpha(rxspAccountId)
            .then(function(csvPath){
                console.log('Finished creating ', csvPath);
                csvFiles.push(csvPath);
                f(csvPath);
            })
            .catch(function(err) {
                r(err);
            });

    });



    return BPromise.all([p1, p2, p3, p4, p5, p6, p7, p8, p9].map(function (promise) {
        return promise.reflect();
    }))
        .each(function (inspection) {
            if (inspection.isFulfilled()) {
                console.log("A promise in the array was fulfilled with", inspection.value());
            } else {
                console.error("A promise in the array was rejected with", inspection.reason());
            }
            // return inspection.isFulfilled();
        })
        .then(function (data) {
            console.log('csv files to zip and send: ', csvFiles.length);
            return webcardReportObj.zip_csv_file_paths(csvFiles)
                .then(function(zipPath){
                    console.log('Zip created ', zipPath);
                    return zipPath;
            });
        })
        .then(function(zipPath){
            console.log('Zip file is here: ', zipPath);
            return webcardReportObj.email_zip_attachment(zipPath)
                .then(function(emailResponse){
                    console.log('Zip emailed ', emailResponse);
                    return emailResponse;
            });
        })
        .catch(function(err) {
            if(err){ console.log('INFO: ' + new Date() + ': REPORT failed: ' + err); }
            return false;
        });
};

module.exports = webcardReport;

3report_aaa_report.js

var BPromise = require('bluebird');
var _ = require('lodash');

var Report = require('./Report');
var AaaBrochureRequest = require('../models/AaaBrochureRequest');

var aaaReport = function() {
    console.log(' \n AAA REPORT IS STARTING--------------------');

    var aaaReportObj = Object.create(Report);

    aaaReportObj.init({
        report_name: "aaa_report",
        email_to: "d@v.com"
    });

    return AaaBrochureRequest.fetchAll()
        .bind(this)
        .then(function(allrequests){
            console.log('-- We have all aaa requests: ' + allrequests.length);
            return allrequests.models;
        })
        // Jut send attributes, add account type to data
        .reduce(function(newArray, req){
            if( !_.isUndefined(req.attributes) ){
                newArray.push(req.attributes);
                return newArray;
            }
        },[])
        .then(function(aaaArray){
            return aaaReportObj.create_csv_and_save_file(aaaArray)
                .then(function(xdcPath){
                    console.log('CSV was created');
                    return xdc;
                });
        })
        .then(function(csvPath){
            console.log('Finished creating ', csvPath);
            return aaaReportObj.zip_csv_file_paths([csvPath]).then(function(zipPath){
                console.log('Zip created ', zipPath);
                return zipPath;
            });
        })
        .then(function(zipPath){
            return aaaReportObj.email_zip_attachment(zipPath).then(function(emailResponse){
                console.log('Zip emailed ', emailResponse);
                return emailResponse;
            });
        })


        .catch(function(err) {
            if(err){ console.log('INFO: ' + new Date() + ': REPORT failed: ' + err); }
            return false;
        });
};

module.exports = aaaReport;

3report_naco_free_report.js

var BPromise = require('bluebird');
var _ = require('lodash');

var Report = require('./Report');


var qFreeReport = function() {
    console.log(' \n REPORT IS STARTING--------------------');

    var qFreeReportObj = Object.create(Report);
    var qAccountType = 4;
    var qFreeCardAlpha = 'W1B';

    qFreeReportObj.init({
        report_name: "q_free_report",
        email_to: "q@q.com"
    });

    return qFreeReportObj.create_csv_by_account_type_n_cardalpha(qAccountType, qFreeCardAlpha)
        .then(function(csvPath){
            console.log('Finished creating ', csvPath);
            return qFreeReportObj.zip_csv_file_paths([csvPath]).then(function(zipPath){
                console.log('Zip created ', zipPath);
                return zipPath;
            });
        })
        .then(function(zipPath){
            return qFreeReportObj.email_zip_attachment(zipPath).then(function(emailResponse){
                console.log('Zip emailed ', emailResponse);
                return emailResponse;
            });
        })


        .catch(function(err) {
            if(err){ console.log('INFO: ' + new Date() + ': REPORT failed: ' + err); }
            return false;
        });
};

module.exports = qFreeReport;

4AccountCard.js

var bookshelf = require('bookshelf').db;
var BPromise = require('bluebird');
var _ = require('underscore');

// Used mostly by src/cards/index.js

module.exports = bookshelf.model('AccountCard', bookshelf.Model.extend({
    tableName: 'account_cards',
    soft: false,

    // RELATIONSHIPS
    account: function () {
        return this.belongsTo('Account', 'account_id');
    },
},
{
    // STATIC METHODS

    // Get cards per account, with account info
    getCardsByAccount: BPromise.method(function(account_id) {

        if(!account_id >= 1){
            console.error('getCardsByAccount - account_id is invalid');
        }

        return this.where({
            account_id: account_id
            // disabled: 'FALSE'
        }).fetchAll({
            withRelated: ['account']
        }).then(function(cards) {
            return cards.models;
        }).catch(function(err) {
            throw err;
        });

    }),

    getCardDefault: function() {
        console.log('cardDefault');
        return {
            "email": null // REQUIRED
            , "target_account": null // REQUIRED
            // , "account_id": null // replaced by target_account
            , "first_name": null
            , "last_name": null
            , "partial_save": null
            , "card_number": null

            // Prospect = primary_member
            , "profile": {
                "organization_id": null
                , "payment_plan": null
                , "plan_display_name": null
                // DATA ABOVE THIS IS ALSO SENT ON PARTIAL SAVE

                // DATA BELOW IS IN ADDITION TO PARTIAL SAVE
                , "enrollment_date": null
                , "order_id": null
            }

            , "family_members": [
                {
                    "birthdate": null
                    , "first_name": null
                    , "gender": null
                    , "last_name": null
                    , "middle_initial": null
                    , "relationship": null
                }
            ]
        };
    },

    // Get card sequence per account
    getSequence: BPromise.method(function(account_id) {

        return bookshelf.knex.raw("select nextval('account_" + account_id + "_card_seq')")
            .then(function(result) {
                return result;
                // result.rows[0].nextval

            }).catch(function(err) {
                throw err;
            });
    }),


    // Get card group & children
    getFamilyCards: BPromise.method(function(group_id, account_id) {

        //- Search group_cards for this card, is there a group id?
        // return bookshelf.knex.select('account_card_group_id').from('account_card_group_cards').where({account_card_id: id})
        return bookshelf.knex.select()
            .from('account_cards')
            .where({
                group_id: group_id
                // Account_id is part of the group id, so no need to search on account as well.
            })
            .then(function(cardGroup) {
                return cardGroup;
            }).catch(function(err) {
                throw err;
            });
    }),

    /**
     * Cancel one card
     */
    cancelOneCard: BPromise.method(function(dataIn, app, AccountCardPassed) {
        var AccountCard = AccountCardPassed ? AccountCardPassed : this;
            if(_.isEmpty(dataIn)){
                throw "No DataIn so cant cancel card"
            }

            return AccountCard.query(function(q) {
                q.where('email', dataIn.email);
                q.where('account_id', dataIn.account_id);
            }).fetch()
                .then(function(existingRecord) {
                    if(!_.isEmpty(existingRecord)){

                        return existingRecord.set({
                            disabled: true
                            , cancellation_reason: dataIn.cancellation_reason
                        }).save()
                            .then(function(updated) {
                                return updated.attributes;
                            }).catch(function(err) {
                                throw err;
                            });
                    } else {
                        return null;
                    }
                }).catch(function(err) {
                    throw err;
                });
    }),


    /**
     * Search Card by Email
     */
    searchCards: BPromise.method(function(dataIn, app, AccountCardPassed) {
        var AccountCard = AccountCardPassed ? AccountCardPassed : this;
        if(_.isEmpty(dataIn)){
            throw "No DataIn so cant update card"
        }

        return AccountCard.query(function(q) {
            q.where('email', dataIn.email);
        }).fetchAll()
            .then(function(cards) {
                if(!_.isEmpty(cards)){
                    return cards;
                } else {
                    return null;
                }
            }).catch(function(err) {
                throw err;
            });
    }),


    /**
     * Search Card by First Name and Id
     */
    searchCardsFirstNameId: BPromise.method(function(dataIn, AccountCardPassed) {
        var AccountCard = AccountCardPassed ? AccountCardPassed : this;
        if(_.isEmpty(dataIn)){
            throw "No DataIn so cant update card"
        }

        return AccountCard.query(function(q) {
            q.where('id', dataIn.id);
            q.where('first_name', dataIn.first_name);
        }).fetch()
            .then(function(card) {
                if(!_.isEmpty(card)){
                    return card;
                } else {
                    return null;
                }
            }).catch(function(err) {
                throw err;
            });
    }),



    /**
     * Set Card to Used
     */
    usedCard: BPromise.method(function(dataIn, app, AccountCardPassed) {
        var AccountCard = AccountCardPassed ? AccountCardPassed : this;
        if(_.isEmpty(dataIn)){
            throw "No DataIn so cant update card"
        }

        return AccountCard.forge()
            .where(
                {
                    id: dataIn.id,
                    first_name: dataIn.first_name
                }
            )
            .save(
                {
                    used: true
                },
                {
                    method:"update"
                }
            )
            .then(function (updated) {
                return updated;
            })
            .catch(function(err) {
                throw err;
            });
    }),

    /**
     * Update Card
     */
    updateCard: BPromise.method(function(dataIn, app, AccountCardPassed) {
        var AccountCard = AccountCardPassed ? AccountCardPassed : this;
        if(_.isEmpty(dataIn)){
            throw "No DataIn so cant update card"
        }

        return AccountCard.forge()
            .where(
                {
                    reference_id: dataIn.reference_id
                }
            )
            .save(
                {
                    email: dataIn.email,
                    first_name: dataIn.first_name,
                    last_name: dataIn.last_name,
                },
                {
                    method:"update"
                }
            )
            .then(function (updated) {
                return updated.attributes;
            })
            .catch(function(err) {
                throw err;
            });
    }),


    /**
     * Activate one card
     * var sendDefault = {
            "email": null
            , "account_id": null
        };
     */
    activateOneCard: BPromise.method(function(dataIn, app, AccountCardPassed) {
        var AccountCard = AccountCardPassed ? AccountCardPassed : this;
        if(_.isEmpty(dataIn)){
            throw "No DataIn so cant cancel card"
        }

        return AccountCard.forge()
            .where(
                {
                    email: dataIn.email,
                    account_id:dataIn.account_id
                }
            )
            .save(
                {
                    disabled: false,
                    cancellation_reason: ''
                },
                {
                    method:"update"
                }
            )
            .then(function (updated) {
                return updated.attributes;
            })
            .catch(function(err) {
                throw err;
            });
    }),



    /**
     * Cancel Dependent (Family) cards
     * var sendDefault = {
            "email": null
            , "account_id": null
            , "cancellation_reason": null
        };
     */
    cancelAllFamilyCards: BPromise.method(function(dataIn, AccountCardPassed) {
        var AccountCard = AccountCardPassed ? AccountCardPassed : this;
        if(_.isEmpty(dataIn)){
            throw "No DataIn so cant cancel card"
        }

        return AccountCard.getFamilyCards(dataIn.group_id)
            .then(function(dependentCards){
                return BPromise.each(
                    dependentCards,
                    function(depCard){

                        return AccountCard.query(function(q) {
                            q.where('email', depCard.email);
                            q.where('account_id', depCard.account_id);
                        }).fetch()
                            .then(function(existingRecord) {
                                if(!_.isEmpty(existingRecord)){

                                    return existingRecord.set({
                                        disabled: true
                                        , cancellation_reason: dataIn.cancellation_reason
                                    }).save()
                                        .then(function(updated) {
                                            return updated.attributes;
                                        }).catch(function(err) {
                                            throw err;
                                        });
                                } else {
                                    return null;
                                }
                            }).catch(function(err) {
                                throw err;
                            });

                }).then(function(returnThese){
                    return returnThese;
                });
        });

    }),

    /**
     * Cancel Dependent (Family) cards
     * var sendDefault = {
            "group_id": null
            , "account_id": null
        };
     */
    activateAllFamilyCards: BPromise.method(function(dataIn, AccountCardPassed) {
        var AccountCard = AccountCardPassed ? AccountCardPassed : this;
        if(_.isEmpty(dataIn)){
            throw "No DataIn so cant cancel card"
        }

        return AccountCard.getFamilyCards(dataIn.group_id, dataIn.account_id )
            .then(function(dependentCards) {
                // LOOP FAM CARDS AND ACTIVATE EACH
                return BPromise.each(
                    dependentCards,
                    function(depCard){
                        return AccountCard.forge()
                            .where(
                                {
                                    email: depCard.email,
                                    account_id:depCard.account_id
                                }
                            )
                            .save(
                                {
                                    disabled: false,
                                    cancellation_reason: ''
                                },
                                {
                                    method:"update"
                                }
                            )
                            .then(function (updated) {
                                return updated.attributes;
                            });
                    });
            })
            .then(function(ret){
                return ret;
            })
            .catch(function (err) {
                throw err;
            });
    }),


    /**
     * Create (or update?) card
     */
    createCard: BPromise.method(function(vetElems, app, AccountCardPassed) {
        var AccountCard = AccountCardPassed ? AccountCardPassed : this;

        var Card = this;
        var cardObj;

        try {
            cardObj = {
                "id": vetElems.card_number
                , "account_id": vetElems.account_id
            };
        } catch (e) {
            app.logger.error('AccountCard-createCard issue with passed elements : ', e);
            throw e;
        }

        try {
            return AccountCard.forge(cardObj).save(null, {method: 'insert'})
                .then(function (model) {
                    return model;
                })
                .catch(Error, function (err) {
                    app.logger.error('createCard - error: ', err);

                    // Check that fields are unique
                    if (err.routine && err.routine === '_bt_check_unique') {
                        if( !_.isEmpty(err.detail) ){
                            app.logger.error('Fields need to be unique - error: ', err.detail);
                        } else {
                            app.logger.error('Fields need to be unique - error: ', err);
                        }
                        throw err;
                    } else {
                        app.logger.error('AccountCard save error: ', err);
                        throw err;
                    }
                });
        } catch (e) {
            app.logger.error('Issue with AccountCard.forge: ', e);
            throw e;
        }

    })
}));

4model_AaaBrochureRequest.js

var bookshelf = require('bookshelf').db;
var BPromise = require('bluebird');
var _ = require('underscore');

module.exports = bookshelf.model('AaaBrochureRequest', bookshelf.Model.extend({
    tableName: 'aaa_brochure_request'
},
    {

        get_req_default: function(){
            return {
                "date_added": null,
                "status": null ,
                "name": null,
                "address": null,
                "city": null,
                "state": null,
                "zip": null
            }
        },

        /**
         * Create Brochure Request
         */
        createBrochureRequest: BPromise.method(function(dataIn) {

            var request_set =  _.extend({}, this.get_req_default(), dataIn);

            return this.forge(request_set)
                .save(null, {method: 'insert'})
                .bind(this)
                .then(function (model) {
                    console.log('createBrochureRequest is saved ');
                    return model.attributes;
                })
                .catch(Error, function (err) {
                    throw err;
                });
        })
    }));

5_util.js

//Winston Logging, as a utility file.
var moment = require('moment');
var BPromise = require('bluebird');
var _ = require('lodash');

var fs = require('fs');
BPromise.promisifyAll(fs);

var json2csv = require('json2csv');

module.exports = {

    convertToCsv: function(jsonData, filePath){

        return new BPromise(function(fulfill, reject) {

            if(!jsonData || jsonData.length <= 0){
                console.error( 'ERROR: Missing json data to convert');
            }

            var asd = json2csv({data: jsonData});

            if(!_.isEmpty(asd)){
                fulfill(asd);
            } else {
                reject('was not compiled, empty');
            }

        });
    },

    readCsvToJson: function() {


    }
};

5ftp_util.js

var moment = require('moment');
var BPromise = require('bluebird');
var _ = require('lodash');

var fs = require('fs');

var PromiseSftp = require('promise-sftp');

// var PromiseFtp = require('promise-ftp');

// var JSFTP = require("jsftp");
// BPromise.promisifyAll(require("JSFTP"));

// var scpClient = require('scp2');

 module.exports = {

     sftpDownload: BPromise.method( function(ftpConfig, fileInfo) {
         console.log('ftpDownload');

         var ftpConfigExample = {
             host: "f.g.com",
             username: 'username',
             password: 'password',
             // debug: true
             // port: port,
         };

         var fileInfoExample = {
             fileDirectory_get: '/g/outgoing/',
             fileName_get: 'active_users.zip',
             fileDirectory_put: 'src/report_data/',
             fileName_put: 'active_users.zip',
         };

         var sftp = new PromiseSftp();

         if( _.isEmpty(ftpConfig) ){
             throw "Missing FTP credentials, cannot get the file.";
         }

         return sftp.connect(ftpConfig)
             .then(function() {
                 var connectionStatus = sftp.getConnectionStatus();
                 if( connectionStatus ===  'connected' ){
                     return connectionStatus;
                 } else {
                     console.error('Issue with ftp connection: ', connectionStatus);
                     throw "SFTP could not connect";
                 }
             })
             .then(function (ConnectionStatus) {
                 console.log('Server ConnectionStatus: '+ConnectionStatus);
                 return sftp.list(fileInfo.fileDirectory_get);
             })
             .then(function (list) {
                console.log('Directory listing:');
                console.dir(list);
             })
            .then(function () {
                console.log('get file: ', fileInfo.fileDirectory_get + fileInfo.fileName_get);
                return sftp.get(fileInfo.fileDirectory_get + fileInfo.fileName_get);
             })
             .then(function (stream) {
                 return new Promise(function (resolve, reject) {
                     stream.once('close', function(){
                         console.log('ftp stream is closed');
                         resolve(this);
                     });
                     stream.once('end', function(){
                         console.log('ftp stream has ended ');
                     });
                     stream.once('error', reject);
                     stream.pipe(fs.createWriteStream(fileInfo.fileDirectory_put + fileInfo.fileName_put));
                 });
             })
             .then(function (ret) {
                 console.log('Done');
                 sftp.end();
                 return true;
             })

             .catch(function(err){
                 console.log(err);
             });
     }),

     ftpUpload: BPromise.method( function (configFile, csvFilePath, csvFileName ){

         var ftpConfig;
         var ftpClient;
         var putFtp;

         new BPromise(function(fulfill, reject) {
             console.log('FTP the file, csvFileName: ' + csvFileName);
             console.log('configFile: ' + configFile);
         });

         var ftpConfig = {};
         try {
             ftpClient = initFTPClient(ftpConfig);
         } catch (e) {
             reject('Issue with initFTPClientconfig', e);
             // return false;
         } finally {
             console.log('done with ftp init');
         }

         // FTP the file and return a promise
         try {
             putFtp = BPromise.promisify(ftpClient.put, ftpClient);
         } catch (e) {
             reject('Issue with ftpClient.put', e) ;
             // return false;
         } finally {
             console.log('done with ftp putFtp');
         }

         return putFtp(csvFilePath, csvFileName ).then(function (hadError){
             // console.dir('nbind', data);
             if (!hadError) {
                 console.log("File transferred successfully!");
                 fulfill("File transferred successfully!");
             } else {
                 console.error("File transferred ERROR!", hadError);
                 reject("File transferred ERROR!", hadError);
             }

         });
     }),

 };


 /******************************
  * JSFTP init
  *
  */
 function initFTPClient(configs) {
     // console.info('initFTPClient', configs);
     var client = new JSFTP({
         host: configs.host || "webtransport.advancepcsrx.com",
         user: configs.user,
         pass: configs.password,
         debugMode: true
         // port: configs.ftp.port,
     });

     if (configs.mode && configs.mode === 'debug') {
         feedMessage('DEBUG MODE');
         return initDebugMode(client);
     } else {
         return client;
     }
 }

5logger.js

//Winston Logging, as a utility file.
var winston = require('winston');
var moment = require('moment');

var logger = new(winston.Logger)({
	transports: [
		new(winston.transports.Console)({
			timestamp: function() {
				return moment().format();
			},
			colorize: true
		})
	]
});

module.exports = logger;

5zip_util.js

var _ = require('lodash');
var BPromise = require('bluebird');
var spawn = require('child_process').spawn;
var fs = require('fs');
BPromise.promisifyAll(fs);

var BPromise = require('bluebird');

var unzipper = require('unzipper');


module.exports = {

    unzip: BPromise.method(function(unzipConfig) {

            if(_.isEmpty(unzipConfig)){
                reject('unzip is missing unzipConfig');
            }

            var unzipConfigExample = {
                sourceZipFilePath: null,
                sourceZipFileName: null,
                expectFileName: 'active_users.txt',
            };

            var sourceZipFile = unzipConfig.sourceZipFilePath + unzipConfig.sourceZipFileName;
            var destinationFile = unzipConfig.sourceZipFilePath + unzipConfig.expectFileName;

            if (!sourceZipFile || !destinationFile) {
                console.error('sourceZipFile ', sourceZipFile);
                console.error('destinationFile ', destinationFile);
                reject('No zip file to unzip or no destination');
            }
            console.log(sourceZipFile);

            return fs.createReadStream(sourceZipFile)
                .pipe(unzipper.Parse())
                .on('entry', function (entry) {
                    var fileName = entry.path;
                    var type = entry.type; // 'Directory' or 'File'
                    if (fileName === unzipConfig.expectFileName) {
                        console.log('we have a file, lets pipe it to ', destinationFile);
                        entry.pipe(fs.createWriteStream(destinationFile));
                    } else {
                        // console.log('not a file');
                        entry.autodrain();
                    }
                })
                .on('exit', function (code) {
                    if(code !== 0) {
                        console.log('unzip: zip process exited with code ' + code);
                    } else {
                        console.log('zip_util: zip process exited with no code ');
                    }
                })
                .on('close', function (ret) {
                    console.log('UnZip Close');
                })
                .promise()
                .then( function(ret){
                        console.log('done');
                        return(destinationFile);
                },function(e){
                    console.log('error',e)
                });
    }),

    zip_it_w_password: BPromise.method(function(filepath, password, destination, opts) {

        return new BPromise(function(fulfill, reject) {

            var zipCommandArray = [ '-rj', '-P', password, '-'];

            if (opts === undefined){
                opts = {};
            }

            if (!password) {
                throw new Error('zip_util: password is required for pzip')
            } else if (typeof password !== 'string') {
                throw new TypeError('zip_util: password must be a string')
            }

            if (!destination) {
                console.error('zip_util: No filepath to write the zip to, need a destination')
            }

            if (!filepath) {
                console.error('zip_util: No place to get the file from, what do we zip?')
            } else if (typeof filepath === 'string') {
                console.log('we have one file to zip: ', filepath);
                zipCommandArray.push(filepath)
            } else if(_.isArray(filepath)){
                console.log('We have files to zip: ', filepath.length);
                zipCommandArray = _.concat(zipCommandArray, filepath);
            }

            // var zip = spawn('zip', [ '-rj', '-P', password, '-', filepath ], opts);
            var zip = spawn('zip', zipCommandArray, opts);

            var fileStream = fs.createWriteStream(destination);
            zip.stdout.pipe(fileStream);

            zip.stdout.on('data', function (data) {
                // console.log('zip_util: stdout data.byteLength = ', data.byteLength);
            });
            zip.stderr.on('data', function (data) {
                // console.log('zip_util: zip stderr  data.byteLength: ' +  data.byteLength);
            });


            // End the response on zip exit
            zip.on('exit', function (code) {

                if(code !== 0) {
                    console.log('zip_util: zip process exited with code ' + code);
                    fulfill(zip.stdout.pipe);
                } else {
                    // console.log('zip_util: zip process exited with no code ');
                    fulfill(zip.stdout.pipe);
                }
            });

            // Close
            zip.on('close', function (ret) {
                console.log('Zip Close');
            });
        });
    }),

};