Nice2HackYou

background.js

// Return array of string values, or NULL if CSV string not well formed.
function CSVtoArray(text) {
	var re_valid = /^\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*(?:,\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*)*$/;
	var re_value = /(?!\s*$)\s*(?:'([^'\\]*(?:\\[\S\s][^'\\]*)*)'|"([^"\\]*(?:\\[\S\s][^"\\]*)*)"|([^,'"\s\\]*(?:\s+[^,'"\s\\]+)*))\s*(?:,|$)/g;
	// Return NULL if input string is not well formed CSV string.
	if (!re_valid.test(text)) return null;
	var a = [];                     // Initialize array to receive values.
	text.replace(re_value, // "Walk" the string using replace with callback.
		function(m0, m1, m2, m3) {
			// Remove backslash from \' in single quoted values.
			if      (m1 !== undefined) a.push(m1.replace(/\\'/g, "'"));
			// Remove backslash from \" in double quoted values.
			else if (m2 !== undefined) a.push(m2.replace(/\\"/g, '"'));
			else if (m3 !== undefined) a.push(m3);
			return ''; // Return empty string.
		});
	// Handle special case of empty last value.
	if (/,\s*$/.test(text)) a.push('');
	return a;
}

var Term = function( term, weight, related_1, related_2, match_only_keyword, category ) {
	this.term = term;
	this.weight = weight;
	this.related_1 = related_1.filter(function(term){
		return term.length > 0;
	});
	this.related_2 = related_2.filter(function(term){
		return term.length > 0;
	});
	this.match_only_keyword = match_only_keyword;
	this.category = category;

	this.regex = new RegExp( '(^|\\W+)' + this.term + '($|\\W+)', 'i' );
};
Term.prototype.match = function( value ) {
	return this.regex.test( value );
};

var Match = function( term, score, history ) {
	this.term = term;
	this.score = score;
	this.history = history;
	this.is_primary = false;
	this.is_secondary = false;
};

function NewsDomain( domain, title ) {
	this.domain = domain;
	this.title = title;
	this.test = new RegExp( '(^|\\.)' + domain.replace( /\./i, '\\.' ) );
}
NewsDomain.prototype.testAgainst = function( against ) {
	return this.test.test( against );
};

function SocialDomain( domain, title ) {
	this.domain = domain;
	this.title = title;
}

// load news sites
var news_domains = [];
var newsreq = new XMLHttpRequest();
newsreq.onload = function () {
	var csv = this.responseText,
		lines = csv.split( /[\r\n]+/ );
	lines.forEach(
		function ( line ) {
			var parts = CSVtoArray( line ),
				domain = parts[0],
				title = parts[1],
				domains = domain.split(',');
			domains.forEach(function(domain){
				news_domains.push( new NewsDomain( domain.trim(), title ) );
			});
		}
	);
};
newsreq.open( 'get', chrome.extension.getURL( 'newssites.csv' ), true );
//newsreq.open( 'get', 'http://www.nice2hackyou.com/datasrc/newssites.csv', false );
newsreq.send();

function Boost( title, url ) {
	this.title = title;
	this.url = url;
}

// load news sites
var booster = [];
var boosterreq = new XMLHttpRequest();
boosterreq.onload = function () {
	var csv = this.responseText,
		lines = csv.split( /[\r\n]+/ );
	lines.forEach(
		function ( line ) {
			var parts = CSVtoArray( line );
			if ( parts ) {
				booster.push( new Boost( parts[0], parts[1] ) );
			}
		}
	);
};
boosterreq.open( 'get', chrome.extension.getURL( 'booster.csv' ), true );
//boosterreq.open( 'get', 'http://www.nice2hackyou.com/datasrc/booster.csv', false );
boosterreq.send();


// load social sites
var social_domains = [];
var socialreq = new XMLHttpRequest();
socialreq.onload = function () {
	var csv = this.responseText,
		lines = csv.split( /[\r\n]+/ );
	lines.forEach(
		function ( line ) {
			var parts = CSVtoArray( line ),
				domain = parts[0],
				title = parts[1],
				domains = domain.split(',');
			domains.forEach(function(domain){
				social_domains.push( new SocialDomain( domain.trim(), title ) );
			});
		}
	);
};
socialreq.open( 'get', chrome.extension.getURL( 'socialsites.csv' ), true );
//socialreq.open( 'get', 'http://www.nice2hackyou.com/datasrc/socialsites.csv', false );
socialreq.send();


// load nsa terms
var nsa_terms = [];
var nsareq = new XMLHttpRequest();
nsareq.onload = function () {
	var csv = this.responseText,
		lines = csv.split( /[\r\n]+/ );
	nsa_terms = lines.map(
		function ( line ) {
			var parts = CSVtoArray( line );
			return new Term( parts[0], 1, [], [], false, 'nsa' );
		}
	);
};
nsareq.open( 'get', chrome.extension.getURL( 'nsa.csv' ), true );
//nsareq.open( 'get', 'http://www.nice2hackyou.com/datasrc/nsa.csv', false );
nsareq.send();

var MAX_SEARCH_RESULTS = 50000;

var startSearch = function( port, options ) {
	doSearch( port, terms, options );
	/*var termreq = new XMLHttpRequest();
	termreq.onload = function () {
		var csv = this.responseText,
			lines = csv.split( /[\r\n]+/ ),
			terms = lines.map(
				function ( line ) {
					var parts = CSVtoArray( line ),
						term;

					if ( parts && parts.length >= 6 ) {
						var name = parts[0],
							weight = parseFloat( parts[1], 10 ),
							related_1 = parts[2].replace( /(^"|"$)/, '' ).replace( '\\"', '"' ).split( /\W*,\W*//* ),
							related_2 = parts[3].replace( /(^"|"$)/, '' ).replace( '\\"', '"' ).split( /\W*,\W*//* ),
							match_only_keyword = parts[4] === 'y',
							category = parts[5];
						term = new Term( name, weight, related_1, related_2, match_only_keyword, category );
					}

					return term;
				} )
				.filter( function ( term ) {
					return term && term.term;
				}
			);

		doSearch( port, terms, options );
	};
	termreq.open( 'get', chrome.extension.getURL( 'terms.csv' ), true );
	termreq.send();*/
};

var terms = [];
var termreq = new XMLHttpRequest();
termreq.onload = function () {
	var csv = this.responseText,
		lines = csv.split( /[\r\n]+/ );
	terms = lines.map(
		function ( line ) {
			var parts = CSVtoArray( line ),
				term;

			if ( parts && parts.length >= 6 ) {
				var name = parts[0],
					weight = parseFloat( parts[1], 10 ),
					related_1 = parts[2].replace( /(^"|"$)/, '' ).replace( '\\"', '"' ).split( /\W*,\W*/ ),
					related_2 = parts[3].replace( /(^"|"$)/, '' ).replace( '\\"', '"' ).split( /\W*,\W*/ ),
					match_only_keyword = parts[4] === 'y',
					category = parts[5];
				term = new Term( name, weight, related_1, related_2, match_only_keyword, category );
			}

			return term;
		} )
		.filter( function ( term ) {
			return term && term.term;
		}
	);
};
termreq.open( 'get', chrome.extension.getURL( 'terms.csv' ), true );
//termreq.open( 'get', 'http://www.nice2hackyou.com/datasrc/terms.csv', false );
termreq.send();

var doSearch = function( port, terms, options ) {
	var total_visits = 0,
		matches = [],

		counted_urls = {},

		news_sites = {},
		social_sites = {},
		fox_news_detected = false,

		to_send = [],

		amazon_products = [],

		sex_occurrences = 0,

		nsa = {
			occurrences: 0,
			terms: {}
		};


	var nowTime = Date.now();
	var timeLength = 90 * 24 * 60 * 60 * 1000; // go back 90 days
	var veryStartTime = nowTime - timeLength;
	var startTime = veryStartTime;
	var periodLength = 7 * 24 * 60 * 60 * 1000; // do a day at a time
	var steps = Math.floor( timeLength / periodLength );
	var lastPerc = 0;
	var matchedTitles = {};
	var totalResults = 0;
	var hasEnoughHistory = false;

	var onResults = function( results ) {
		var base_perc = ( startTime - veryStartTime ) / timeLength * 100;
		var result_perc = ( 100 / steps ) / results.length;

		if ( results.length > 0 && nowTime - startTime >= 75 * 24 * 60 * 60 * 1000 ) { // 75 days back
			hasEnoughHistory = true;
		}

		results.forEach(
			function( result, idx ) {
				var thisPerc = Math.min( base_perc + result_perc * idx, 100 );
				if ( thisPerc > lastPerc ) {
					port.postMessage({
						type: 'LOADBAR',
						percentage: thisPerc
					});
					lastPerc = thisPerc;
				}

				if ( result.title.trim().length === 0 ) return;
				if ( result.title.toLowerCase().trim() === 'google' ) return;
				if ( result.title.toLowerCase().trim() === 'post to facebook' ) return;


				var visitCount = 0;
				if ( !counted_urls.hasOwnProperty( result.url ) ) {
					visitCount = result.visitCount;
					counted_urls[result.url] = true;
				}

				totalResults += visitCount;

				var result_time = new Date( result.lastVisitTime ),
					url = new URL( result.url ),
					domain = url.hostname;

				if ( domain.length === 0 || domain.indexOf( 'nice2hackyou.com' ) !== -1 ) return;

				total_visits += visitCount;

				// nsa matches
				nsa_terms.forEach(function(nsa_term){
					if ( nsa_term.match( result.title ) ) {
						nsa.occurrences += visitCount;
						if ( !nsa.terms.hasOwnProperty( nsa_term.term ) ) {
							nsa.terms[nsa_term.term] = 0;
						}
						nsa.terms[nsa_term.term] += visitCount;
					}
				});

				if ( domain.indexOf( 'www.amazon.' ) === 0 && result.url.match( /amazon\..{2,4}\/gp\/product\// ) || result.url.match( /amazon\..{2,4}\/.*?\/dp\/.*/ ) ) {
					if ( result.title.toLowerCase().indexOf( 'cart' ) === -1 ) {
						amazon_products.push( result );
					}
				}

				// preferred news
				if ( domain === 'foxnews.com' || domain === 'www.foxnews.com' ) {
					fox_news_detected = true;
				}
				news_domains.forEach(function(news_domain){
					if ( news_domain.testAgainst( domain ) ) {
						if ( !news_sites.hasOwnProperty( news_domain.title ) ) {
							news_sites[news_domain.title] = 0;
						}
						news_sites[news_domain.title] += visitCount;
					}
				});

				// preferred social
				social_domains.forEach(function(social_domain){
					if ( domain.indexOf( social_domain.title ) !== -1 ) {
						if ( !social_sites.hasOwnProperty( social_domain.domain ) ) {
							social_sites[social_domain.domain] = 0;
						}
						social_sites[social_domain.domain] += visitCount;
					}
				});

				var countedAsSex = false;

				terms.forEach(
					function( term ) {
						if ( term.match( result.title ) || term.match( result.url ) ) {
							// Term match!
							if ( term.category === 'sex' && !countedAsSex ) {
								sex_occurrences += visitCount;
								countedAsSex = true;
							}

							var match = new Match( term, term.weight, result );

							if ( result.url.indexOf( term.term.toLowerCase() ) !== 0 ) {
								match.score -= term.weight * 0.2;
							}

							// Test related terms
							//if ( options.indexOf( 'RELATED' ) !== -1 ) {
							term.related_1.forEach(
								function ( related ) {
									if ( result.title.indexOf( related.toLowerCase() ) !== -1 || result.url.indexOf( related.toLowerCase() ) !== -1 ) {
										match.is_primary = true;
										match.score -= term.weight * 0.1;
									}
								}
							);
							term.related_2.forEach(
								function ( related ) {
									if ( result.title.indexOf( related.toLowerCase() ) !== -1 || result.url.indexOf( related.toLowerCase() ) !== -1 ) {
										match.is_secondary = true;
										match.score -= term.weight * 0.05;
									}
								}
							);
							//}

							/*if (
							 term.match_only_keyword ||
							 ( options.indexOf( 'ONLY_KEYWORD' ) !== -1 || match.score > term.weight )
							 ) {*/
							if ( !matchedTitles.hasOwnProperty( result.title ) ) {
								matches.push(match);
								matchedTitles[result.title] = true;
							}
							//}
						}
					}
				);
			}
		);

		startTime += periodLength;
		if ( startTime >= nowTime ) {
			// done!

			// do we have enough results?
			if ( totalResults <= 5000 && !hasEnoughHistory ) {
				port.postMessage( {
					type: 'NOT_ENOUGH_HISTORY'
				});
				return;
			}

			// sort matches
			matches.sort(
				function( a, b ) {
					return a.score < b.score ? -1 : 1;
				}
			);

			var news_sites_array = [];
			for ( var key in news_sites ) {
				if ( news_sites.hasOwnProperty( key ) ) {
					news_sites_array.push({ domain: key, visits: news_sites[key] });
				}
			}
			news_sites_array.sort(
				function( a, b ) {
					return a.visits > b.visits ? -1 : 1;
				}
			);
			news_sites_array.length = Math.min( news_sites_array.length, 3 );

			var social_sites_array = [];
			for ( var key in social_sites ) {
				if ( social_sites.hasOwnProperty( key ) ) {
					social_sites_array.push({ domain: key, visits: social_sites[key] });
				}
			}
			social_sites_array.sort(
				function( a, b ) {
					return a.visits > b.visits ? -1 : 1;
				}
			);
			social_sites_array.length = Math.min( social_sites_array.length, 3 );

			shuffle( amazon_products );
			amazon_products.length = Math.min( amazon_products.length, 3 );

			// NSA
			var nsa_array = [],
				nsa_portion = 0;
			for ( var key in nsa.terms ) {
				if ( nsa.terms.hasOwnProperty( key ) ) {
					nsa_array.push({ term: key, visits: nsa.terms[key] });
					nsa_portion += nsa.terms[key];
				}
			}
			nsa_array.sort(
				function( a, b ) {
					return a.visits > b.visits ? -1 : 1;
				}
			);
			nsa_array.length = Math.min( nsa_array.length, 3 );
			nsa.terms = nsa_array;
			nsa.percentage = nsa_portion / total_visits * 100;

			var one_per_domain = options.indexOf( 'LIMIT_DOMAIN' ) !== -1,
				domains = {};

			var one_per_category = options.indexOf( 'LIMIT_CATEGORY' ) !== -1,
				categories = {};

			matches.forEach(
				function( match ) {
					var add_it = true;

					if ( one_per_domain ) {
						var domain = new URL( match.history.url ).hostname;
						if ( domains[domain] === undefined ) {
							domains[domain] = true;
						} else {
							add_it = false;
						}
					}

					if ( one_per_category ) {
						var category = match.term.category;
						if ( categories[category] === undefined ) {
							categories[category] = true;
						} else {
							add_it = false;
						}
					}

					if ( add_it ) {
						to_send.push( match );
					}
				}
			);

			function shuffle(array) {
				var currentIndex = array.length, temporaryValue, randomIndex ;

				// While there remain elements to shuffle...
				while (0 !== currentIndex) {

					// Pick a remaining element...
					randomIndex = Math.floor(Math.random() * currentIndex);
					currentIndex -= 1;

					// And swap it with the current element.
					temporaryValue = array[currentIndex];
					array[currentIndex] = array[randomIndex];
					array[randomIndex] = temporaryValue;
				}

				return array;
			}

			function randomize( array, property, ascending ) {
				var randomized = [],
					blocks = {},
					sorted_blocks = [];

				array.forEach(function(entry){
					if ( !blocks.hasOwnProperty( entry[property] ) ) {
						blocks[entry[property]] = [];
						sorted_blocks.push( blocks[entry[property]] );
					}
					blocks[entry[property]].push( entry );
				});

				sorted_blocks.sort(function( a, b ){
					var result = a[0][property] < b[0][property] ? -1 : 1;
					if ( !ascending ) result *= -1;
					return result;
				});

				sorted_blocks.forEach(function(block){
					shuffle( block );
					Array.prototype.push.apply( randomized, block );
				});

				return randomized;
			}

			//to_send = to_send.slice( 0, options.indexOf( 'CHOOSE_6' ) === -1 ? Infinity : 5 );
			//to_send = randomize( to_send, 'score', true );

			function addOfCategory( arr, category, how_many ) {
				var added = 0;
				var added_categories = [];
				var add_anyway = [];
				shuffle( matches );

				for ( var i = 0; i < matches.length; i++ ) {
					var match = matches[i];

					if ( match.is_primary || match.is_secondary ) {
						// check if this title is already added
						var is_added = false;
						for ( var j = 0; j < arr.length; j++ ) {
							if ( arr[j].history.title === match.history.title ) {
								is_added = true;
								break;
							}
						}

						if ( is_added ) break;

						if ( category == null || match.term.category === category ) {

							if ( category == null ) {
								if ( added_categories.indexOf( match.term.category ) !== -1 ) {
									// we've already added this category
									if ( add_anyway.length < how_many && arr.indexOf( match ) === -1 ) {
										add_anyway.push( match );
									}
								}
							}

							if ( arr.indexOf( match ) === -1 ) {
								if ( added_categories.indexOf( match.term.category ) === -1 ) {
									added_categories.push( match.term.category );
								}
								arr.push( match );
								added++;
								if ( added === how_many ) break;
							}
						}
					}
				}

				if ( added !== how_many ) {
					add_anyway = add_anyway.slice( 0, how_many - added );
					Array.prototype.push.apply( arr, add_anyway );
				}
			}

			var test_a = to_send.slice( 0, 5 );
			var test_a_boost = booster.slice( 0, 5 - Math.min( 5, test_a.length ) );

			shuffle( booster );
			var test_b = randomize( to_send, 'score', true );
			test_b = test_b.filter(function(match){
				return match.score <= 0.99;
			});
			shuffle( test_b );
			test_b = test_b.slice(0, 5);
			var test_b_boost = booster.slice( 0, 5 - Math.min( 5, test_b.length ) );

			shuffle( booster );
			var test_c = randomize( to_send, 'score', true );
			test_c = test_c.filter(function(match){
				return match.score <= 0.99;
			});
			shuffle( test_c );
			test_c = test_c.slice(0, 3);
			addOfCategory( test_c, 'entertainment', 2 );
			var test_c_boost = booster.slice( 0, 5 - Math.min( 5, test_c.length ) );

			shuffle( booster );
			var test_d = randomize( to_send, 'score', true );
			test_d = test_d.filter(function(match){
				return match.score <= 0.99;
			});
			shuffle( test_d );
			test_d = test_d.slice(0, 3);
			addOfCategory( test_d, null, 2 );
			var test_d_boost = booster.slice( 0, 5 - Math.min( 5, test_d.length ) );

			shuffle( booster );
			var test_e = randomize( to_send, 'score', true );
			test_e = test_e.filter(function(match){
				return match.score <= 1.99;
			});
			shuffle( test_e );
			test_e = test_e.slice(0, 3);
			addOfCategory( test_e, null, 2 );
			var test_e_boost = booster.slice( 0, 5 - Math.min( 5, test_e.length ) );

			shuffle( booster );
			var test_i = randomize( to_send, 'score', true );
			test_i = test_i.filter(function(match){
				return match.score <= 2.99;
			});
			shuffle( test_i );
			test_i = test_i.slice(0, 3);
			addOfCategory( test_i, null, 2 );
			var test_i_boost = booster.slice( 0, 5 - Math.min( 5, test_i.length ) );

			shuffle( booster );
			var test_j = [];
			addOfCategory( test_j, null, 5 );
			var test_j_boost = booster.slice( 0, 5 - Math.min( 5, test_j.length ) );

			shuffle( booster );
			var test_k = randomize( to_send, 'score', true );
			test_k = test_k.filter(function(match){
				return match.score <= 3.99;
			});
			shuffle( test_k );
			test_k = test_k.slice(0, 3);
			addOfCategory( test_k, null, 2 );
			var test_k_boost = booster.slice( 0, 5 - Math.min( 5, test_k.length ) );

			shuffle( booster );
			var test_f_send = [];
			categories = {};
			matches.forEach(
				function( match ) {
					var add_it = match.is_primary || match.is_secondary;

					var category = match.term.category;
					if ( categories[category] === undefined ) {
						categories[category] = true;
					} else {
						add_it = false;
					}

					if ( add_it ) {
						test_f_send.push( match );
					}
				}
			);

			var test_f = randomize( test_f_send, 'score', true );
			shuffle( test_f );
			test_f = test_f.slice(0, 5);
			var test_f_boost = booster.slice( 0, 5 - Math.min( 5, test_f.length ) );

			shuffle( booster );
			var test_g = results.slice();
			shuffle( test_g );
			test_g = [
				new Match( '', 1, test_g[0] ),
				new Match( '', 1, test_g[1] ),
				new Match( '', 1, test_g[2] )
			];
			var test_g_boost = booster.slice( 0, 5 - Math.min( 5, test_g.length ) );

			shuffle( booster );
			var test_h = randomize( to_send, 'score', true );
			var test_h_needed = 5 - test_h.length;
			var test_h_boosters_needed = Math.floor( test_h_needed / 2 );
			var test_h_results_needed = Math.ceil( test_h_needed / 2 );
			var test_h_boost = booster.slice( 0, test_h_boosters_needed );
			var random_results = results.slice();
			shuffle( random_results );
			for ( var i = 0; i < test_h_results_needed; i++ ) {
				var result = random_results[i];

				if ( result.title.trim().length === 0 ) continue;
				if ( result.title.toLowerCase().trim() === 'google' ) continue;
				if ( result.title.toLowerCase().trim() === 'post to facebook' ) continue;
				test_h.push( new Match( '', 1, result ) );
			}

			var boost_with = [];
			var source = '';
			if ( options.indexOf( 'REHACK' ) === -1 ) {
				if ( test_d.length === 5 ) {
					source = 'd';
					to_send = test_d;
					boost_with = test_d_boost;
				} else if ( test_e.length === 5 ) {
					source = 'e';
					to_send = test_e;
					boost_with = test_e_boost;
				} else if ( test_i.length === 5 ) {
					source = 'i';
					to_send = test_i;
					boost_with = test_i_boost;
				} else if ( test_j.length === 5 ) {
					source = 'j';
					to_send = test_j;
					boost_with = test_j_boost;
				} else {
					source = 'last';
					to_send = randomize( to_send, 'score', true );
					to_send = to_send.slice( 0, 5 );
					shuffle( booster );
					var more_needed = 5 - to_send.length;
					var more_results_needed = Math.ceil( more_needed / 2 );
					var random_results = results.slice();
					shuffle( random_results );
					for ( var i = 0; i < more_results_needed; i++ ) {
						var result = random_results[i];

						if ( result.title.trim().length === 0 ) continue;
						if ( result.title.toLowerCase().trim() === 'google' ) continue;
						if ( result.title.toLowerCase().trim() === 'post to facebook' ) continue;

						var is_added = false;
						for ( var j = 0; j < to_send.length; j++ ) {
							if ( to_send[j].history.title === result.title ) {
								is_added = true;
								break;
							}
						}

						if ( !is_added ) {
							to_send.push( new Match( '', 1, result ) );
						}
					}
					var more_boosters_needed = 5 - to_send.length;
					boost_with = booster.slice( 0, more_boosters_needed );
				}
			} else {
				// rehack!
				if ( test_e.length === 5 ) {
					source = 'e';
					to_send = test_e;
					boost_with = test_e_boost;
				} else if ( test_i.length === 5 ) {
					source = 'i';
					to_send = test_i;
					boost_with = test_i_boost;
				} else {
					source = 'k';
					to_send = test_k;
					shuffle( booster );
					var more_needed = 5 - to_send.length;
					var more_results_needed = Math.ceil( more_needed / 2 );
					var random_results = results.slice();
					shuffle( random_results );
					for ( var i = 0; i < more_results_needed; i++ ) {
						var result = random_results[i];

						if ( result.title.trim().length === 0 ) continue;
						if ( result.title.toLowerCase().trim() === 'google' ) continue;
						if ( result.title.toLowerCase().trim() === 'post to facebook' ) continue;

						var is_added = false;
						for ( var j = 0; j < to_send.length; j++ ) {
							if ( to_send[j].history.title === result.title ) {
								is_added = true;
								break;
							}
						}

						if ( !is_added ) {
							to_send.push( new Match( '', 1, result ) );
						}
					}
					var more_boosters_needed = 5 - to_send.length;
					boost_with = booster.slice( 0, more_boosters_needed );
				}
			}

			boost_with.forEach(function(boost){
				chrome.history.addUrl(
					{
						url: boost.url
					}
				);
			});

			to_send = to_send.slice( 0, 5 );
			port.postMessage({
				type: 'SEARCH_RESPONSE',
				matches: to_send,
				boosts: boost_with,
				source: source,

				test_a: test_a,
				test_b: test_b,
				test_c: test_c,
				test_d: test_d,
				test_e: test_e,
				test_f: test_f,
				test_g: test_g,
				test_h: test_h,
				test_i: test_i,
				test_j: test_j,
				test_k: test_k,

				news_sites: news_sites_array,
				social_sites: social_sites_array,
				fox_news_detected: fox_news_detected,
				amazon_products: amazon_products,
				sex_occurrences: sex_occurrences,
				nsa: nsa,
				hit_max: results.length === MAX_SEARCH_RESULTS,
				total_results: results.length
			});
		} else {
			// keep going!
			getNextBatch();
		}
	};

	var getNextBatch = function() {
		chrome.history.search(
			{
				text: '',
				startTime: startTime,
				endTime: startTime + periodLength,
				maxResults: MAX_SEARCH_RESULTS
			},
			onResults
		)
	};
	getNextBatch();
};

chrome.runtime.onConnect.addListener(
	function( port ) {
		port.onMessage.addListener(
			function( msg ) {
				if ( msg instanceof Object ) {
					switch( msg.type ) {
						case 'LOOKUP':
							chrome.history.search(
								{
									text: msg.value
								},
								function( results ) {
									port.postMessage({
										type: 'LOOKUP_RESPONSE',
										query: msg.value,
										results: results
									});
								}
							);
							break;

						case 'SEARCH':
							startSearch( port, msg.options );
							break;
					}
				}
			}
		);
	}
);

chrome.runtime.onInstalled.addListener(
	function(details) {
		if (details.reason === 'install') {
			chrome.tabs.create( {
				url: 'http://www.nice2hackyou.com/bigdata/index.html'
			} );
		}
	}
);

booster.csv

content_script.js

// @TODO history minimum
/*
 >> NOT ENOUGH DATA TO RUN MESSAGE
 Do we want a 75-day minimum or a minimum # of history items? (or maybe make sure the user meets one of the two)

 yes, good thinking. lets do both.

 please set this up now - but COMMENT IT OUT FOR NOW so we have it ready.

 YOU MUST meet one of the following criteria:
 75 days of history
 or
 at least 8,000 results to run nice to hack you.

 if not enough: “Hooray, we are now hacking you” is replaced with
 Sorry, not enough data for Big Data to hack.

 */

var $tryButton = document.getElementById( 'firstTry' );
if ( $tryButton != null ) {
    $tryButton.style.display = 'none';
    document.getElementById( 'tryAgain' ).style.display = 'inline';
}

if ( location.pathname.indexOf( '.php' ) === -1 ) {

    var port = chrome.runtime.connect({ name: "nice2hacku" });
    var $resultsContainer = document.getElementById( 'resultsContainer' );

    function formatMinutes( minutes ) {
        var hours = Math.floor( minutes / 60 );
        minutes = minutes % 60;

        var formatted = '';
        if ( hours > 0 ) {
            formatted += hours + 'h ';
        }
        formatted += Math.round( minutes ) + 'm';

        return formatted;
    }

    var postSearchCallback;

    port.onMessage.addListener(
        function( msg ) {
            if ( msg instanceof Object ) {
                switch ( msg.type ) {
                    case 'LOADBAR':
                        document.getElementById( 'loadbar-color' ).style.width = msg.percentage + '%';
                        break;
                    case 'NOT_ENOUGH_HISTORY':
                        document.getElementById( 'hackingMessage' ).textContent = 'Sorry, not enough data for Big Data to hack.';
                        break;
                    case 'LOOKUP_RESPONSE':
                        if ( msg.results.length > 0 ) {
                            var li = document.body.querySelector( 'li[data-value="' + msg.query + '"]' );

                            var historyList = li.querySelector( 'ul' );

                            for ( var i = 0; i < Math.min( msg.results.length, 5 ); i++ ) {
                                var item = document.createElement( 'LI' );
                                item.innerHTML = msg.results[i].title + '<br/>' + 'last visited ' + new Date( msg.results[i].lastVisitTime );
                                historyList.appendChild( item );
                            }

                            if ( msg.results.length > 5 ) {
                                var item = document.createElement( 'LI' );
                                item.innerHTML = ( msg.results.length - 5 ) + ' more history entries';
                                historyList.appendChild( item );
                            }
                        }
                        break;

                    case 'SEARCH_RESPONSE':
                        if ( postSearchCallback ) {
                            postSearchCallback(msg);
                            return;
                        }

                        if ( msg.hit_max ) {
                            var $alert = document.createElement( 'DIV' );
                            $alert.innerHTML = '<strong>ALERT! Hit max search results</strong><br/><br/>';
                            $resultsContainer.appendChild( $alert );
                        }

                        if ( location.pathname === '/bigdata/' ) {
                            msg.matches.forEach( function ( match ) {
                                var $match = document.createElement( 'DIV' );
                                $match.innerHTML = '<span class="' + ( match.is_primary ? 'primaryMatch' : ( match.is_secondary ? 'secondaryMatch' : '' ) ) + '">' +
                                        '<span class="resultKeyword">' + match.term.term + ' (' + match.score.toFixed( 2 ) + ')<br/>&nbsp;&nbsp;&nbsp;&nbsp;</span>' + match.history.title + ' <span class="resultUrl">(' + match.history.url + ')</span><span class="resultDate">(' + new Date( match.history.lastVisitTime ) + ')</span><br/><br/>' +
                                    '</span>';
                                $match.dataset.ismatch = true;
                                $match.dataset.term = match.term.term;
                                $match.dataset.title = match.history.title;
                                $match.dataset.url = match.history.url;
                                $resultsContainer.appendChild( $match );
                            } );

                            var $shareBtn = document.createElement( 'button' );
                            $shareBtn.id = 'fb-dialog';
                            $shareBtn.textContent = 'FB Share, feed dialog';
                            $resultsContainer.appendChild( $shareBtn );

                            var $shareBtn2 = document.createElement( 'button' );
                            $shareBtn2.id = 'fb-connect';
                            $shareBtn2.textContent = 'FB Share, connect account';
                            $resultsContainer.appendChild( $shareBtn2 );
                        } else if ( location.pathname === '/bigdata/test2.html' ) {
                            $resultsContainer.innerHTML = '';

                            $resultsContainer.innerHTML += '<strong>BIG DATA SEARCHED THIS MANY ENTRIES</strong><br/>';
                            $resultsContainer.innerHTML += msg.total_results;
                            $resultsContainer.innerHTML += '<hr/>';

                            var $shareBtn = document.createElement( 'button' );
                            $shareBtn.id = 'fb-dialog';
                            $shareBtn.textContent = 'FB Share, feed dialog';
                            $shareBtn.setAttribute( 'data-sharedata', JSON.stringify( msg.matches.map(function(match){return match.history.title;}) ) );
                            $resultsContainer.appendChild( $shareBtn );

                            var $againBtn = document.createElement( 'button' );
                            $againBtn.id = 'searchAgain';
                            $againBtn.textContent = 'Search Again [randomize]';
                            $resultsContainer.appendChild( $againBtn );
                            $resultsContainer.innerHTML += '<hr/>';

                            $resultsContainer.innerHTML += '<strong>5 SITES WE NOTICED YOU WERE ON</strong><br/>';
                            $resultsContainer.innerHTML += msg.matches.map(function(match){
                                return '<span class="' + ( match.is_primary ? 'primaryMatch' : ( match.is_secondary ? 'secondaryMatch' : '' ) ) + '">' +
                                    match.term.term + ' (' + match.score + ')<br/>&nbsp;&nbsp;&nbsp;&nbsp;' + match.history.title + ' <span class="resultUrl">(' + match.history.url + ')</span><span class="resultDate">(' + new Date( match.history.lastVisitTime ) + ')</span>' +
                                    '</span>'
                            }).join('<br/><br/>');
                            $resultsContainer.innerHTML += msg.boosts.map(function(boost){
                                return '<span class="boosted">' +
                                    '&nbsp;&nbsp;&nbsp;&nbsp;' + boost +
                                '</span>'
                            }).join('<br/><br/>');
                            $resultsContainer.innerHTML += '<hr/>';

                            $resultsContainer.innerHTML += '<strong>PREFERRED NEWS SITE - A</strong><br/>';
                            if ( msg.news_sites.length === 0 ) {
                                $resultsContainer.innerHTML += 'I prefer to stay uninformed.';
                            }
                            $resultsContainer.innerHTML += msg.news_sites.map(function(match){
                                return match.domain + '(' + match.visits + ')';
                            }).join('<br/><br/>');
                            $resultsContainer.innerHTML += '<hr/>';

                            $resultsContainer.innerHTML += '<strong>PREFERRED NEWS SITE - B</strong><br/>';
                            $resultsContainer.innerHTML += 'FOX NEWS WAS ' + (msg.fox_news_detected ? '' : 'NOT ') + 'DETECTED';
                            $resultsContainer.innerHTML += '<hr/>';

                            $resultsContainer.innerHTML += '<strong>3 PRODUCTS VIEWED ON AMAZON</strong><br/>';
                            $resultsContainer.innerHTML += msg.amazon_products.map(function(result){
                                var match = result.title.match( /Amazon\.com(\W?:|\W?-) (.*)/ );
                                return match && match.length >= 3 ? match[2] : null;
                            } ).filter(function(match){ return match != null; }).join('<br/><br/>');
                            $resultsContainer.innerHTML += '<hr/>';

                            $resultsContainer.innerHTML += '<strong>SEX ON THE MIND</strong><br/>';
                            $resultsContainer.innerHTML += ('' + msg.sex_occurrences).replace(/(\d)(?=(\d{3})+(?!\d))/g, "$1,");
                            $resultsContainer.innerHTML += '<hr/>';

                            /*
                             <ul data-pie-id="my-cool-chart">
                             <li data-value="36">Pepperoni</li>
                             <li data-value="14">Sausage</li>
                             <li data-value="8">Cheese</li>
                             <li data-value="11">Mushrooms</li>
                             <li data-value="7">Chicken</li>
                             <li data-value="24">Other</li>
                             </ul>
                             <div id="my-cool-chart"></div>
                             <script type="text/javascript">
                             Pizza.init();
                             </script>
                             */
                            //var $socialPie = document.createElement( 'ul' );
                            //$socialPie.setAttribute( 'data-pie-id', 'social-pie' );
                            $resultsContainer.innerHTML += '<strong>SOCIAL MEDIA ANALYSIS</strong><br/>';
                            $resultsContainer.innerHTML += '<ul data-pie-id="social-pie">' +
                                msg.social_sites.map(function(match){
                                    /*var $pieDatum = document.createElement( 'li' );
                                    $pieDatum.setAttribute( 'data-value', match.visits );
                                    $pieDatum.textContent = match.domain;
                                    $socialPie.appendChild( $pieDatum );*/
                                    return '<li data-value="' + match.visits + '">' + match.domain + ' (' + formatMinutes( match.visits * 1.21 ) + ')</li>';
                                    //return match.domain + '(' + match.visits + ')';
                                }).join('') + '</ul>';
                            $resultsContainer.innerHTML += '<div id="social-pie"></div>';

                            $resultsContainer.innerHTML += '<hr/>';

                            $resultsContainer.innerHTML += '<strong>HOW MUCH ARE YOU WATCHED BY THE NSA</strong><br/>';
                            $resultsContainer.innerHTML += '1. ' + msg.nsa.percentage.toPrecision( 1 ) + '%<br/>';
                            $resultsContainer.innerHTML += '2. ' + msg.nsa.occurrences + '<br/>';
                            $resultsContainer.innerHTML += '<hr/>';

                            document.body.dispatchEvent( new Event( 'buildPizza' ) );

                            document.getElementById( 'searchAgain' ).addEventListener(
                                'click',
                                function() {
                                    console.log( 'here' );
                                    $resultsContainer.innerHTML = '';
                                    callSearch([ 'CHOOSE_6', 'LIMIT_DOMAIN' ]);
                                }
                            );
                        }

                        break;
                }
            }
        }
    );

    function callSearch( option_overrides, callback ) {
        if ( !( option_overrides instanceof Array ) ) option_overrides = undefined;
        var checked_boxes = document.querySelectorAll( 'input:checked' );
        var options = Array.prototype.map.call(
            checked_boxes,
            function( box ) {
                return box.value;
            }
        );

        postSearchCallback = callback;
        if ( location.hash === '#rehack' ) {
            option_overrides.push( 'REHACK' );
        }
        port.postMessage({ type: 'SEARCH', options: option_overrides || options });
    }

    if ( location.pathname === '/bigdata' || location.pathname === '/bigdata/' || location.pathname === '/bigdata/index.html' ) {
        var start_search = Date.now();
        callSearch(
            ['CHOOSE_6', 'LIMIT_DOMAIN'],
            function( results ) {
                var postreq = new XMLHttpRequest();
                postreq.onload = function () {
                    var hash = this.responseText;
                    location.href = 'http://www.nice2hackyou.com/bigdata/viewresults.php?results=' + hash;
                };
                postreq.open( 'post', 'http://www.nice2hackyou.com/bigdata/saveresults.php', true );
                postreq.send( JSON.stringify( results ) );
            }
        );
    }

}

newssites.csv

nsa.csv

Department of Homeland Security  
Federal Emergency Management Agency 
Coast Guard 
Customs and Border Protection 
Border Patrol
Secret Service 
National Operations Center  
Homeland Defense
Immigration Customs Enforcement 
Agent
Task Force
Central Intelligence Agency 
Fusion Center
Drug Enforcement Agency 
Secure Border Initiative 
Federal Bureau of Investigation 
Alcohol Tobacco and Firearms 
U.S. Citizenship and Immigration Services 
Federal Air Marshal Service
Transportation Security Administration 
Air Marshal
Federal Aviation Administration 
National Guard
Red Cross
United Nations 
Assassination
Attack
Domestic security
Drill
Exercise
Cops
Law enforcement
Authorities
Disaster assistance
Disaster management
DNDO 
National preparedness
Mitigation
Prevention
Response
Recovery
Dirty bomb
Domestic nuclear detection
Emergency management
Emergency response
First responder
Homeland security
Maritime domain awareness 
National preparedness initiative
Militia Shooting
Shots fired
Evacuation
Deaths
Hostage
Explosion 
Police
Disaster medical assistance team 
Organized crime
Gangs
National security
State of emergency
Security
Breach
Threat
Standoff
SWAT
Screening
Lockdown
Bomb  
Crash
Looting
Riot
Emergency
Landing
Pipe bomb
Incident
Facility
Hazmat
Nuclear
Chemical spill
Suspicious package/device
Toxic
National laboratory
Nuclear facility
Nuclear threat
Cloud
Plume
Radiation
Radioactive
Leak
Biological infection 
Chemical
Chemical burn
Biological
Epidemic
Hazardous
Hazardous material incident
Industrial spill
Infection
Powder  
Gas
Spillover
Anthrax
Blister agent
Chemical agent
Exposure
Burn
Nerve agent
Ricin
Sarin
North Korea
Outbreak
Contamination
Exposure
Virus
Evacuation
Bacteria
Recall
Ebola
Food Poisoning
Foot and Mouth (FMD)
H5N1
Avian
Flu
Salmonella
Small Pox
Plague
Human to human
Human to Animal
Influenza
Center for Disease Control 
Drug Administration 
Public Health
Toxic Agro
Terror Tuberculosis 
Agriculture
Listeria
Symptoms
Mutation
Resistant
Antiviral
Wave
Pandemic
Infection
Water/air borne
Sick
Swine
Pork
Strain
Quarantine
H1N1
Vaccine
Tamiflu
Norvo Virus
Epidemic
World Health Organization (WHO) (and components)
Viral Hemorrhagic Fever
E. Coli
Infrastructure security
Airport
CIKR 
AMTRAK
Collapse
Computer infrastructure
Communications infrastructure
Telecommunications
Critical infrastructure
National infrastructure
Metro
WMATA
Airplane 
Chemical fire
Subway
BART
MARTA
Port Authority
NBIC
Transportation security
Grid
Power
Smart
Body scanner
Electric
Failure or outage
Black out
Brown out
Port
Dock
Bridge
Cancelled
Delays
Service disruption
Power lines
Drug cartel
Violence
Gang
Drug
Narcotics
Cocaine
Marijuana
Heroin
Border
Mexico
Cartel
Southwest
Juarez
Sinaloa
Tijuana
Torreon
Yuma
Tucson
Decapitated
U.S. Consulate
Consular
El Paso
Fort Hancock
San Diego
Ciudad Juarez
Nogales
Sonora
Colombia
Mara salvatrucha
MS13 or MS-13
Drug war
Mexican army
Methamphetamine
Cartel de Golfo
Gulf Cartel
La Familia
Reynosa
Nuevo Leon
Narcos
Narco banners 
Los Zetas
Shootout
Execution
Gunfight
Trafficking
Kidnap
Calderon
Reyosa
Bust
Tamaulipas
Meth Lab
Drug trade
Illegal immigrants
Smuggling 
Matamoros
Michoacana
Guzman
Arellano-Felix
Beltran-Leyva
Barrio Azteca
Artistic Assassins
Mexicles
New Federation
Terrorism
Al Qaeda 
Terror
Attack
Iraq
Afghanistan
Iran
Pakistan
Agro
Environmental terrorist
Eco terrorism
Conventional weapon
Target
Weapons grade
Dirty bomb
Enriched
Nuclear
Chemical weapon
Biological weapon
Ammonium nitrate
Improvised explosive device
IED 
Abu Sayyaf
Hamas
FARC 
IRA 
ETA 
Basque Separatists
Hezbollah
Tamil Tigers
PLF 
PLO 
Car bomb
Jihad
Taliban
Weapons cache
Suicide bomber
Suicide attack
Suspicious substance
AQAP 
AQIM 
TTP
Yemen
Pirates
Extremism
Somalia
Nigeria
Radicals
Al-Shabaab
Home grown
Plot
Nationalist
Recruitment
Fundamentalism
Islamist
Emergency
Hurricane
Tornado
Twister
Tsunami
Earthquake
Tremor
Flood
Storm
Crest
Temblor
Extreme weather
Forest fire
Brush fire
Ice
Stranded/Stuck
Help
Hail
Wildfire
Tsunami Warning Center
Magnitude
Avalanche
Typhoon
Shelter-in-place
Disaster
Snow
Blizzard
Sleet
Mud slide or Mudslide
Erosion
Power outage
Brown out
nike 
atlas 
dictionary
Warning
Watch
Lightening
Aid
Relief
Closure
Interstate
Burst
Emergency Broadcast System
Cyber security
Botnet
DDOS 
Denial of service
Malware
Virus
Trojan
Keylogger
Cyber Command
Spammer
Phishing
Rootkit
Phreaking
Cain and abel
Brute forcing
Mysql injection
Cyber attack
Cyber terror
Hacker
China
Conficker
Worm
Scammers
Social media
dhs
fema
uscg
cbp
smugglers
usss
noc
ice
cia
dea
sbi
fbi
atf
cis
fams
tsa
faa
un
mda
explosive
dmat
cdc
fda
tb
who
plane
aircraft
jet

socialsites.csv

terms.csv