Skip to content

Instantly share code, notes, and snippets.

@bitprophet
Created August 19, 2011 01:57
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save bitprophet/198cbc3d7b7b8d9a37d3 to your computer and use it in GitHub Desktop.
Save bitprophet/198cbc3d7b7b8d9a37d3 to your computer and use it in GitHub Desktop.
updated for python 2.5 and 2.4
diff --git a/fabric/main.py b/fabric/main.py
index 5139a45..de9aa8b 100644
--- a/fabric/main.py
+++ b/fabric/main.py
@@ -14,14 +14,13 @@ from optparse import OptionParser
import os
import sys
-from fabric import api # For checking callables against the API
+from fabric import api # For checking callables against the API
from fabric.contrib import console, files, project # Ditto
from fabric.network import denormalize, interpret_host_string, disconnect_all
from fabric import state # For easily-mockable access to roles, env and etc
from fabric.state import commands, connections, env_options
from fabric.utils import abort, indent
-
# One-time calculation of "all internal callables" to avoid doing this on every
# check of a given fabfile callable (in is_task()).
_modules = [api, project, files, console]
@@ -439,6 +438,21 @@ def main():
commands[r] = lambda: api.run(remainder_command)
commands_to_run.append((r, [], {}, [], []))
+
+ if state.env.run_in_parallel:
+ #We want to try to import the multiprocessing module by default.
+ #The state is checked to see if it was specifically requested, and
+ #in that case an error is reported.
+ try:
+ import multiprocessing
+
+ except ImportError:
+ state.env.run_in_parallel = False
+ print("Unable to run in parallel as requested.\n" +
+ "You need the multiprocessing module.\n" +
+ "Continuing.")
+
+
# At this point all commands must exist, so execute them in order.
for name, args, kwargs, cli_hosts, cli_roles in commands_to_run:
# Get callable by itself
@@ -448,6 +462,8 @@ def main():
# Set host list (also copy to env)
state.env.all_hosts = hosts = get_hosts(
command, cli_hosts, cli_roles)
+
+ jobs = []
# If hosts found, execute the function on each host in turn
for host in hosts:
# Preserve user
@@ -458,9 +474,26 @@ def main():
if state.output.running:
print("[%s] Executing task '%s'" % (host, name))
# Actually run command
- commands[name](*args, **kwargs)
+
+ if state.env.run_in_parallel:
+ p = multiprocessing.Process(
+ target = commands[name],
+ args = args,
+ kwargs = kwargs,
+ )
+ jobs.append(p)
+ p.start()
+ else:
+ commands[name](*args,**kwargs)
+
# Put old user back
state.env.user = prev_user
+
+ #only runs if was set to run in parallel, and causes fabric to
+ #wait to end program until all Processes have returned.
+ for p in jobs:
+ p.join()
+
# If no hosts found, assume local-only and run once
if not hosts:
commands[name](*args, **kwargs)
diff --git a/fabric/state.py b/fabric/state.py
index 5dcd4d4..18e2268 100644
--- a/fabric/state.py
+++ b/fabric/state.py
@@ -186,7 +186,15 @@ env_options = [
action='store_true',
default=False,
help="force use of pseudo-terminal in run/sudo"
- )
+ ),
+
+ # Parallel execution model flag
+ make_option('-P', '--parallel',
+ dest='run_in_parallel',
+ action='store_true',
+ default=False,
+ help="use the multiprocessing module to fork by hosts"
+ ),
]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment