Graphite data migration : ./migrate_graphite_data.py -u <user> -f <folder (/opt/graphite/storage/whisper/)> -s <source_host (graphite01-integration.fstrz.net)> -d <dest_host (monitoring-integration.fstrz.net)>

migrate_graphite_data.py

#!/usr/bin/python

import sys, getopt, time
from subprocess import *

def main(argv):
  source_host = ''
  dest_host = ''
  user = ''
  folder = ''

  try:
    opts, args = getopt.getopt(argv,"hu:f:b:s:d:",["user=","folder=","sourcehost=","desthost="])
  except getopt.GetoptError:
    print 'migrate_graphite_data.py -u <user> -f <folder> -s <sourcehost> -d <desthost>'
    sys.exit(2)
  for opt, arg in opts:
    if opt == '-h':
      print 'migrate_graphite_data.py -u <user> -f <folder> -s <sourcehost> -d <desthost>'
      sys.exit()
    elif opt in ("-f", "--folder"):
      folder = arg
    elif opt in ("-u", "--user"):
      user = arg
    elif opt in ("-s", "--sourcehost"):
      source_host = arg
    elif opt in ("-d", "--desthost"):
      dest_host = arg
  print 'Args : '
  print ' - User :', user
  print ' - Folder :', folder
  print ' - Source host :', source_host
  print ' - Destination host :', dest_host
  print '\n'
  migrate_whisper_files(user, folder, source_host, dest_host)

def migrate_whisper_files(user, folder, source_host, dest_host):
  print "Retrieving file list from", source_host
  p = Popen('ssh ' + user + '@' + source_host + ' "sudo find ' + folder + ' | grep .wsp"', shell=True, stdout=PIPE)
  source_files = p.stdout.readlines()
  print "File list retrieved from", source_host
  print "\n"

  print "Retrieving file list from", dest_host
  p = Popen('ssh ' + user + '@' + dest_host + ' "sudo find ' + folder + ' | grep .wsp"', shell=True, stdout=PIPE)
  dest_files = p.stdout.readlines()
  print "File list retrieved from", dest_host
  print "\n"

  for common_files in [set(source_files).intersection(dest_files)]:
    if len(common_files) > 0:
      i = 0
      metric_files = []
      for common_file in common_files:
        print "File detected :", common_file.replace('\n', '')
        i += 1
        metric_files.append(common_file.replace('\n', ''))
        if i > 0 and (i%500 == 0 or i == len(common_files)):
          tar_name = 'migration_' + str(int(time.time())) + '.tar.gz'
          migrate_files(user, folder, source_host, dest_host, tar_name, metric_files)
          metric_files = []
    else:
      print "No file to migrate"

def migrate_files(user, folder, source_host, dest_host, tar_name, metric_files):
  print "\nPacking metrics files into " + tar_name + " on " + source_host + " " + user + " home"
  p = Popen('ssh ' + user + '@' + source_host + ' -o StrictHostKeyChecking=no "tar -cvzf ' + tar_name + ' ' + ' '.join(metric_files) + '"', shell=True)
  p.wait()
  print "Copying " + tar_name + " on " + dest_host + " " + user + " home"
  p = Popen('ssh ' + user + '@' + source_host + ' -o StrictHostKeyChecking=no "scp ' + tar_name + ' ' + user + '@' + dest_host + ':' + tar_name + '"', shell=True)
  p.wait()
  print "Unpacking " + tar_name + " on " + dest_host + " folder " + folder
  p = Popen('ssh ' + user + '@' + dest_host + ' -o StrictHostKeyChecking=no "sudo tar -C / --overwrite -zxvf ' + tar_name + '"', shell=True)
  p.wait()
  print "Removing " + tar_name + " on " + dest_host + " " + user + " home"
  p = Popen('ssh ' + user + '@' + dest_host + ' -o StrictHostKeyChecking=no "rm ' + tar_name + '"', shell=True)
  p.wait()
  for metric_file in metric_files:
    print "Removing metric file " + metric_file + " on " + source_host
    p = Popen('ssh ' + user + '@' + source_host + ' -o StrictHostKeyChecking=no "sudo rm ' + metric_file + '"', shell=True)
    p.wait()

if __name__ == "__main__":
   main(sys.argv[1:])