Skip to content

Instantly share code, notes, and snippets.

@spacecowboy
Created September 8, 2011 16:04
Show Gist options
  • Star 3 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save spacecowboy/1203760 to your computer and use it in GitHub Desktop.
Save spacecowboy/1203760 to your computer and use it in GitHub Desktop.
A simple proof of concept for building your own super computer in python, assuming you have access to many machines.
from multiprocessing.managers import BaseManager
from mathenate import mathenate
#Define manager
class QueueManager(BaseManager): pass
QueueManager.register('get_job_queue')
QueueManager.register('get_result_queue')
#Connect to server
m = QueueManager(address=('my.computer.org', 50000), authkey='password')
m.connect()
#Set up queus
job_queue = m.get_job_queue()
result_queue = m.get_result_queue()
#Since we can't be sure that the slaves have time to fill the result queue
#we clear the contents of the queue first
while(not result_queue.empty()):
print("waiting results: " + str(result_queue.get()))
print('sending add-job')
job_queue.put(('add', 2, 3))
print('sending sub-job')
job_queue.put(('sub', 5, 1))
print('sending special-job')
job_queue.put((mathenate, 2, 10))
while(not result_queue.empty()):
print("result = " + str(result_queue.get()))
def mathenate(a, b):
''' Returns a to the power of b. '''
return a**b
from multiprocessing.managers import BaseManager
import Queue
'''
Taken directly from the examples for multiprocessing. The only purpose for this
file is to serve two queues for clients, of which there are two.
'''
#Define two queues, one for putting jobs on, one for putting results on.
job_queue = Queue.Queue()
result_queue = Queue.Queue()
#Define a custom class to manage communication
class QueueManager(BaseManager): pass
QueueManager.register('get_job_queue', callable=lambda:job_queue)
QueueManager.register('get_result_queue', callable=lambda:result_queue)
#Start up
m = QueueManager(address=('my.computer.org', 50000), authkey='password')
s = m.get_server()
s.serve_forever()
from multiprocessing.managers import BaseManager
#Define manager
class QueueManager(BaseManager): pass
QueueManager.register('get_job_queue')
QueueManager.register('get_result_queue')
#Connect to server
m = QueueManager(address=('my.computer.org', 50000), authkey='password')
m.connect()
#Set up queus
job_queue = m.get_job_queue()
result_queue = m.get_result_queue()
while(True):
job = job_queue.get()
print(job)
#Job is always an indexable object
#It can either be a string identifying the operation
#Or it can be a real function object!
#With the caveat that the function must be pickleable
op = job[0]
if op == 'add':
res = job[1] + job[2]
elif op == 'sub':
res = job[1] - job[2]
else:
res = op(job[1], job[2])
print("Sending result: " + str(res))
result_queue.put(res)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment