Skip to content

Instantly share code, notes, and snippets.

@chapter09
Created May 8, 2016 20:33
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save chapter09/e228438195e199fa1117493d69a15d2f to your computer and use it in GitHub Desktop.
Save chapter09/e228438195e199fa1117493d69a15d2f to your computer and use it in GitHub Desktop.
A python script for running rsync in parallel
#!/usr/bin/python
import os
import sys
import subprocess
import argparse
import re
import socket
# argv[1] host lists
# argv[2] source file/directory
# argv[3] dest file/directory
parser = argparse.ArgumentParser()
parser.add_argument("-f", "--host", dest="hostfile",
help="read host list from file", metavar="FILE")
parser.add_argument("-s", "--src", dest="source",
help="source file/directory", metavar="SRC")
parser.add_argument("-d", "--dst", dest="destination",
help="destination file/directory", metavar="DST")
args = parser.parse_args()
if len(sys.argv) != 7:
#print len(sys.argv)
parser.print_help()
parser.exit(0)
host_list = []
host_list_fd = open(args.hostfile)
#print host_list_fd
# regex for IPv4
pat = r"(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})"
for line in host_list_fd.readlines():
m = re.search(pat, line)
if m:
print m.group(0).strip()
host_list.append(m.group(0).strip())
host_list_fd.close()
local_addr = socket.gethostbyname(socket.gethostname())
for host in host_list:
if host.strip() == local_addr.strip():
continue
cmd = "rsync -az %s %s:%s"%(args.source, host, args.destination)
print cmd
p = subprocess.Popen(cmd, shell=True)
p.wait()
print "DONE"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment