Created
June 7, 2018 02:25
-
-
Save michaelguia/ddac0efd671bf9c0a42c734bea0b3801 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
mport threading | |
import time | |
from queue import Queue | |
import sys | |
from os import listdir | |
from collections import Counter | |
q = Queue() | |
directory = sys.argv[1] | |
files = listdir(directory) | |
# print(files) | |
# f = files[0] | |
def create_dict_from_email(path): | |
text = open(path, errors='ignore').read() | |
d = Counter(text.split()) | |
print(threading.current_thread().name) | |
q.put(d) | |
# sequential | |
# for f in files: | |
# d = create_dict_from_email(directory + '/' + f) | |
# print(d) | |
threads = [] | |
for f in files: | |
p = directory + '/' + f | |
t = threading.Thread(target=create_dict_from_email, args=(p,)) | |
threads.append(t) | |
t.start() | |
from functools import reduce | |
results = [] | |
for t in threads: | |
# waits for all threads to finish | |
t.join() | |
results.append(q.get()) | |
#print(results) | |
r = reduce(lambda x,y: x+y, results) | |
print(r) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment