Created
September 8, 2011 16:04
-
-
Save spacecowboy/1203760 to your computer and use it in GitHub Desktop.
A simple proof of concept for building your own super computer in python, assuming you have access to many machines.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from multiprocessing.managers import BaseManager | |
from mathenate import mathenate | |
#Define manager | |
class QueueManager(BaseManager): pass | |
QueueManager.register('get_job_queue') | |
QueueManager.register('get_result_queue') | |
#Connect to server | |
m = QueueManager(address=('my.computer.org', 50000), authkey='password') | |
m.connect() | |
#Set up queus | |
job_queue = m.get_job_queue() | |
result_queue = m.get_result_queue() | |
#Since we can't be sure that the slaves have time to fill the result queue | |
#we clear the contents of the queue first | |
while(not result_queue.empty()): | |
print("waiting results: " + str(result_queue.get())) | |
print('sending add-job') | |
job_queue.put(('add', 2, 3)) | |
print('sending sub-job') | |
job_queue.put(('sub', 5, 1)) | |
print('sending special-job') | |
job_queue.put((mathenate, 2, 10)) | |
while(not result_queue.empty()): | |
print("result = " + str(result_queue.get())) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def mathenate(a, b): | |
''' Returns a to the power of b. ''' | |
return a**b |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from multiprocessing.managers import BaseManager | |
import Queue | |
''' | |
Taken directly from the examples for multiprocessing. The only purpose for this | |
file is to serve two queues for clients, of which there are two. | |
''' | |
#Define two queues, one for putting jobs on, one for putting results on. | |
job_queue = Queue.Queue() | |
result_queue = Queue.Queue() | |
#Define a custom class to manage communication | |
class QueueManager(BaseManager): pass | |
QueueManager.register('get_job_queue', callable=lambda:job_queue) | |
QueueManager.register('get_result_queue', callable=lambda:result_queue) | |
#Start up | |
m = QueueManager(address=('my.computer.org', 50000), authkey='password') | |
s = m.get_server() | |
s.serve_forever() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from multiprocessing.managers import BaseManager | |
#Define manager | |
class QueueManager(BaseManager): pass | |
QueueManager.register('get_job_queue') | |
QueueManager.register('get_result_queue') | |
#Connect to server | |
m = QueueManager(address=('my.computer.org', 50000), authkey='password') | |
m.connect() | |
#Set up queus | |
job_queue = m.get_job_queue() | |
result_queue = m.get_result_queue() | |
while(True): | |
job = job_queue.get() | |
print(job) | |
#Job is always an indexable object | |
#It can either be a string identifying the operation | |
#Or it can be a real function object! | |
#With the caveat that the function must be pickleable | |
op = job[0] | |
if op == 'add': | |
res = job[1] + job[2] | |
elif op == 'sub': | |
res = job[1] - job[2] | |
else: | |
res = op(job[1], job[2]) | |
print("Sending result: " + str(res)) | |
result_queue.put(res) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment