Skip to content

Instantly share code, notes, and snippets.

@paxti
Created March 23, 2018 15:44
Show Gist options
  • Save paxti/4d0ba325eaa19d513ce249611f326b2a to your computer and use it in GitHub Desktop.
Save paxti/4d0ba325eaa19d513ce249611f326b2a to your computer and use it in GitHub Desktop.
Find all prospect in the DB by showName and save to separate files
const MongoClient = require('mongodb').MongoClient
const assert = require('assert')
const Json2csvParser = require('json2csv').Parser
const Promise = require('bluebird')
const fs = Promise.promisifyAll(require('fs'))
const csvdata = require('csvdata')
let database
let collection
const fileWithShows = './data_5.csv'
const fileWithShowsTest = './data/test.csv'
const url = 'mongodb://localhost:27017'
const fields = ["id", "campaign_id", "salutation", "first_name", "last_name", "email", "company", "website", "job_title", "showname", "department", "country", "address_one", "address_two", "city", "state", "territory", "zip", "phone", "fax", "source", "annual_revenue", "employees", "industry", "years_in_business", "comments", "notes", "score", "grade", "last_activity_at", "recent_interaction", "is_do_not_email", "is_do_not_call", "opted_out", "created_at", "updated_at", "TST_Dealer_Intro_Link", "TST_Custom_Banner_URL", "TST_Custom_Banner_Link", "TST_Custom_Goodbye", "TST_Custom_Headline", "TST_Custom_Thanks", "TST_Custom_Welcome", "TST_Custom_Welcome2", "TST_Dealer_Address1", "TST_Dealer_CityStateZip", "TST_Dealer_Contact", "TST_Dealer_Email", "TST_Dealer_Logo_Link", "TST_Dealer_Name", "TST_Dealer_Phone", "TST_Dealer_Website", "TST_Facebook_Link", "TST_Flickr_Link", "TST_LinkedIn_Link", "TST_Owner_Title", "TST_Pinterest_Link", "TST_Twitter_Link", "TST_YouTube_Link"]
const opts = { fields }
MongoClient.connect(url)
.then(client => {
database = client
collection = client.db('myproject').collection('prospects')
return client.db('myproject').collection('prospects')
})
.then(collection => {
return csvdata.load(fileWithShows)
})
.then(shows => {
return shows.map(({ showname }) => {
return Promise.props({ data: collection.find({$or: [{ "showname.value": showname }, { showname: showname }]}), query: showname })
})
})
.then(findPromises => {
return Promise.all(findPromises)
})
.then(cursors => {
return cursors.map(cursor => {
return Promise.props({ data: cursor.data.toArray(), query: cursor.query })
})
})
.then(toArrayPromises => {
return Promise.all(toArrayPromises)
})
.then(collections => {
return collections.map(collection => {
return {
data:collection.data.map(record => {
if (record.showname && record.showname.value) {
return Object.assign(record, {showname: record.showname.value.join(',')})
}
return record
}),
query: collection.query
}
})
})
.then(collections => {
database.close()
return collections.filter(collection => collection.data.length > 10).map(collection => {
const parser = new Json2csvParser(opts)
return { csv: parser.parse(collection.data), query: collection.query }
})
})
.then(objects => {
return objects.map((object, index) => {
return fs.writeFileAsync( `res_from_db/${object.query}_${object.csv.split("\n").length - 1}.csv`, object.csv)
})
})
.then(files => {
return Promise.all(files)
})
.then(res => {
console.log('Done')
})
.catch((err) => {
console.log(err)
})
@paxti
Copy link
Author

paxti commented Mar 23, 2018

Not optimized at all

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment