chrisdlees

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Net;
using System.Text;

/* EXAMPLE USAGE
	FileDownloader fileDownloader = new FileDownloader();

chrisdlees

var fs = require('fs');
var csv = require('fast-csv');
var stream = fs.createReadStream('/home/jaywon/Downloads/modified-salary-per-state-per-jobtitle.csv');
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var masterList = [];
var mongoose = require('mongoose');
mongoose.connect('mongodb://localhost:27017/job-imports');

//define schema for import data 

chrisdlees

# Set up the constants:
$ObjectPronouns = 'Her', 'Him', 'Them'
$PossesivePronouns = 'Her', 'His', 'Their'
$PersonalPronouns = 'She', 'He', 'They'
$States = 'California', 'Texas', 'Florida', 'New York', 'Pennsylvania', 'Illinois', 'Ohio', 'Georgia', 'North Carolina', 'Michigan'
$Nouns = 'Athlete', 'Clown', 'Shovel', 'Paleo Diet', 'Doctor', 'Parent', 'Cat', 'Dog', 'Chicken', 'Robot', 'Video Game', 'Avocado', 'Plastic Straw', 'Serial Killer', 'Telephone Psychic'
$Places = 'House', 'Attic', 'Bank Deposit Box', 'School', 'Basement', 'Workplace', 'Donut Shop', 'Apocalypse Bunker'
$When = 'Soon', 'This Year', 'Later Today', 'RIGHT NOW', 'Next Week'

function generateAreMillennialsKillingHeadline { 

chrisdlees

# The following will split a CSV (file.csv) into multiple parts of 1 million lines each
# with each part having its own header.  
#
# PREFIX denotes the filename to use for the parts.  A number will be added to the end.

tail -n +2 file.csv |
split -d -l 1000000 - --filter='sh -c "{ head -n1 file.csv; cat; } > $FILE"' PREFIX