Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Testing python multiprocessing
from multiprocessing import Pool
import jsonlines
from datetime import datetime
def get_full_text(tweet):
if tweet['truncated']:
return(tweet['extended_tweet']['full_text'])
return (tweet['text'])
def dothings(tweets):
startTime = datetime.now()
results = list(map(get_full_text, tweets))
print('{} (regular map)'.format(datetime.now() - startTime))
startTime = datetime.now()
results2 = [get_full_text(tweet) for tweet in tweets]
print('{} (list comprehension)'.format(datetime.now() - startTime))
def dothings_faster(tweets, number_of_processes):
pool = Pool(processes=number_of_processes)
startTime = datetime.now()
results = pool.map_async(get_full_text, tweets)
results.wait()
print('{} ({} processes)'.format((datetime.now() - startTime), number_of_processes))
if __name__ == '__main__':
startTime = datetime.now()
tweets = []
with jsonlines.open('test-tweets.jsonl') as reader:
for tweet in reader:
tweets.append(tweet)
print('{} (loading Tweets)'.format(datetime.now() - startTime))
print('\n')
dothings(tweets)
dothings_faster(tweets,1)
dothings_faster(tweets,4)
dothings_faster(tweets,8)
dothings_faster(tweets,12)
dothings_faster(tweets,24)
dothings_faster(tweets,48)
dothings_faster(tweets,96)
dothings_faster(tweets,120)
dothings_faster(tweets,168)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.