Sync Script for frontend repo for API project

sync.py

#!/usr/bin/python
import argparse
import datetime
import io
import json
import os
import sys
import urllib
import urllib2

VERSION = '0.0.1'

"""
USAGE: python sync.py -v -u=USERNAME -p=API_KEY -f=True
"""
def main(argv):
   parser = argparse.ArgumentParser()
   parser.add_argument('-v', '--verbosity', action="count",  help="increase output verbosity (e.g., -vv is more than -v)")
   parser.add_argument("-c", "--configuration", type=str, required=False)
   parser.add_argument("-u", "--username", type=str, required=False)
   parser.add_argument("-p", "--password", type=str, required=False)
   parser.add_argument("-f", "--force", type=bool, required=False)
   args = parser.parse_args()
   verbose = args.verbosity

   verbose = True
   
   # -- STEP 1: Read Configuration Variables   
   current_directory = os.path.dirname(os.path.realpath(__file__))
   default_config_file = os.path.join(current_directory, 'sync.json')
   conf_var = raw_input("Specify which config file to use (or leave blank to use %s)"%default_config_file)

   conf_file = conf_var or default_config_file
   if not os.path.exists(conf_file):
      print "Config file not found at %s"%(conf_file)
      var = raw_input("Type enter to close")
      return

   configuration = None
   with open(conf_file, 'r') as conf:
      configuration = json.load(conf)

   if not configuration:
      if verbose:
         print "Config file empty at %s"%(conf_file)
         var = raw_input("Type enter to close")
      return

   api_username = args.username or configuration['api_username']
   api_key = args.password or configuration['api_key']

   media_root = configuration['media_root']
   api_root = configuration['api_root']
   distribution_path = configuration['distribution_path']
      
   if verbose:
      print "media_root: %s"%(media_root)
      print "api_root: %s"%(api_root)
      print "distribution_path: %s"%(distribution_path)      
   
   
   for kiosk in configuration["kiosks"]:
      kiosk_category = kiosk['kiosk_category']
      kiosk_slug = kiosk['kiosk_slug']
      

      if verbose:
         print "================================"
         print "Starting Sync for Kiosk: %s - %s"%(kiosk_category, kiosk_slug)

      # -- STEP 2: Sync JSON to local distribution
      sync_data(api_username, api_key, kiosk_category, kiosk_slug, media_root, api_root, distribution_path, verbose)

      # -- STEP 3: Sync Media to local distribution
      sync_media(api_username, api_key, kiosk_category, kiosk_slug, media_root, api_root, distribution_path, args.force, verbose)

      if verbose:
         print "Sync for %s Complete!"%(kiosk_category)

   var = raw_input("Done! Type enter to close")

def sync_data(api_username, api_key, kiosk_category, kiosk_slug, media_root, api_root, distribution_path, verbose=False):
   config_url = '%skiosk/%s/%s/?format=json&api_key=%s&username=%s'%(api_root, kiosk_category, kiosk_slug, api_key, api_username)
   data_url = '%skiosk/%s/%s/data/?format=json&api_key=%s&username=%s'%(api_root, kiosk_category, kiosk_slug, api_key, api_username)

   config_file_path = os.path.join(distribution_path, "proxy/api/v1/kiosk", kiosk_category, kiosk_slug, "index.php")
   data_file_path = os.path.join(distribution_path, "proxy/api/v1/kiosk", kiosk_category, kiosk_slug, "data", "index.php")

   retrieve_json_file(config_url, config_file_path, verbose)
   retrieve_json_file(data_url, data_file_path, verbose)

   

def retrieve_json_file(target_url, target_file_path, verbose=False):
   if verbose:
      print "Retriving json from %s"%(target_url)

   response = urllib2.urlopen(target_url)
   response_text = response.read()
   timestamp = '{:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())
   data = "<?PHP header('Content-Type: application/json'); /* Generated on %s */ ?> %s"%(timestamp, response_text)
   
   target_file_dir = os.path.dirname(target_file_path)
   if not os.path.exists(target_file_dir):
      os.makedirs(target_file_dir)

   with io.open(target_file_path, 'w', encoding='utf-8') as f:
      f.write(unicode(data))

def sync_media(api_username, api_key, kiosk_category, kiosk_slug, media_root, api_root, distribution_path, force=False, verbose=False):
   url = '%skiosk/%s/%s/media/?format=json&api_key=%s&username=%s'%(api_root, kiosk_category, kiosk_slug, api_key, api_username)

   response = urllib2.urlopen(url)
   response_text = response.read()
   data = json.loads(response_text)
   media = data['media']
   download_media(media, media_root, distribution_path, force, verbose)   

def download_media(media, media_root, download_path, force=False, verbose=False):
   total = len(media)
   ctr = 0
   for media_item in media:
      ctr += 1
      target_file_path = os.path.join(download_path, media_item)         
      load_url = '%s%s'%(media_root, media_item)
      if verbose:
         print 'Downloading %s/%s %s -> %s'%(ctr, total, load_url, target_file_path)
      download_file(load_url, target_file_path, force, verbose)

def download_file(url, local_file_path, force=False, verbose=False):

   #Create enclosing DIR if it doesn't exist
   local_file_dir = os.path.dirname(local_file_path)
   if not os.path.exists(local_file_dir):
      os.makedirs(local_file_dir)

   #Now test if we need to download file
   exists = os.path.exists(local_file_path)

   if exists:
      #If the file has changed, then download it:
      site = urllib.urlopen(url)
      meta = site.info()
      remote_size = int(meta.getheaders("Content-Length")[0])
      remote_modified = datetime.datetime.strptime(meta.getheaders("last-modified")[0], '%a, %d %b %Y %H:%M:%S GMT')      

      f = open(local_file_path, "rb")
      local_size = int(len(f.read()))
      local_modified = datetime.datetime.fromtimestamp(os.path.getmtime(local_file_path))      
      f.close()

      file_size_different = remote_size != local_size
      file_newer = local_modified < remote_modified
            
   else:
      file_size_different = True
      file_newer = True

   do_download = force or exists==False or file_size_different==True or file_newer==True
   if verbose:
      print "--- Download=%s [Force=%s, Exists=%s, File Size Different=%s, File Newer=%s]"%(do_download, force, exists, file_size_different, file_newer)

   if do_download:
      file = urllib.URLopener()
      file.retrieve(url, local_file_path)


if __name__ == "__main__":
   main(sys.argv[1:])