clean_needles_from_haystack.sce

// This was written for Chandni 14-Dec-2011
//
// Problem Statement: 
// 
// Assuming you have two files, d1.txt and d2.txt. You would like to remove all
// entries from d2.txt that also appear in d1.txt


d1 = read_csv('d1.txt');
d2 = read_csv('d2.txt', ', ')';

needle   = stripblanks(d1(:, 1));
haystack = stripblanks(d2);

[needle_in_haystack, which_needle] = grep(haystack, needle);

clean_haystack = haystack;
clean_haystack(needle_in_haystack) = [];

// write_csv(clean_haystack, "cleaned_database.txt")


sorted_needle   = gsort(needle, 'lr', 'i'); 
write_csv(sorted_needle, "sorted_database_already_emailed.txt")

sorted_haystack = gsort(haystack, 'lr', 'i'); 
write_csv(sorted_haystack, "sorted_database_to_email_with_duplicates_from_already_emailed.txt")

sorted_clean_haystack = gsort(clean_haystack, 'lr', 'i'); 
write_csv(sorted_clean_haystack, "sorted_cleaned_database_to_email.txt", ', ')