Export a group of feature classes (stored in ArcSDE) using a specified JSON structured text file as the input parameter. This script will build packaged and unpackaged zipped shape and zipped file geodatabase files, a folder structure, and a .bat file for

Export Utah SGID ArcSDE Data To File Packages.py

#20120725_15:23pm
#working but in need of future refactoring (ex. change .bat to .py?)

#Runs off a locally stored .txt file containing JSON objects
#Writes zipped statewide gdb and shp data to outputRoot
#Writes .bat file with time stamp for replacing existing ftp site content

# jsonPackagesfile is a JSON data structure containing package objects. Each data package object
# consists of:
#   ["Name"] the output name for the package,
#   ["Category"] the category name it will be stored under, a blank means that it will pick up the category from the first feature class,
#   ["ClipOptions"] not yet implemented, will tell whether the package and its contents should be clipped to counties
#   ["FeatureClasses"] the list of featureclasses for the package

# ex JSON text file content:

#   [
#      {
#         "Name": "StateCountyMunicipalBoundaries",
#         "Category": "BOUNDARIES",
#         "ClipOptions": "",
#         "FeatureClasses": [
#             "SGID10.BOUNDARIES.Counties",
#             "SGID10.BOUNDARIES.Municipalities",
#             "SGID10.BOUNDARIES.Municipalities_Carto",
#             "SGID10.BOUNDARIES.Utah"
#         ]
#      }
#   ]


#keep JSON doc current @
#https://docs.google.com/a/utah.gov/document/d/1TFWbHtXifrhZhf8PGR1VrITBnw0AFqFjv1fK-dTMj14/edit

#JSON can be verified at http://jsonlint.com

# **************************
# SET THESE 4 parameters:

jsonPackagesfile = r'C:/sgidout/jsonpackages/jsonPackages_Boundaries_StateCountyMunicipalBoundaries.txt'
sdeConnectionStr = r'Database Connections\dc_agrc@SGID10@gdb10.agrc.utah.gov.sde'
outPathRoot = r'C:\sgidout'
ftpPathRoot = r'J:\UtahSGID_Vector\UTM12_NAD83'

# **************************

import arcpy, zipfile, zlib, os, json, datetime
from time import strftime

arcpy.env.overwriteOutput = True

def clipToCounties(inFC, clipFC, outputDir):

    #placeholder for future clipping functionality
    print "Clip To Counties STUB"

def zipws(path, zip, keep):
    path = os.path.normpath(path)
    # os.walk visits every subdirectory, returning a 3-tuple
    #  of directory name, subdirectories in it, and file names
    #  in it.

    for (dirpath, dirnames, filenames) in os.walk(path):
        # Iterate over every file name
        #
        for file in filenames:
            # Ignore .lock files
            #
            if not file.endswith('.lock'):
                arcpy.AddMessage("Adding %s..." % os.path.join(path, dirpath, file))
                try:
                    if keep:
                        if os.path.join(dirpath, file).find('.zip') == -1:
                            zip.write(os.path.join(dirpath, file), os.path.join(os.path.basename(path), os.path.join(dirpath, file)[len(path)+len(os.sep):]))

                    else:
                        if os.path.join(dirpath, file).find('gdb') == -1 and os.path.join(dirpath, file).find('.zip') == -1:
                            #print 'DP    ' + dirpath
                            #print 'file0 ' + file.split(".")[0]
                            #print 'full  ' + dirpath + file.split(".")[0] + '\\' + file

                            # file to write, location to write
                            #zip.write(os.path.join(dirpath, file), os.path.join(dirpath[len(path):], file))
                            zip.write(os.path.join(dirpath, file), file.split(".")[0] + '\\' + file)

                except Exception, e:
                    arcpy.AddWarning("    Error adding %s: %s" % (file, e))

    return None

# MAIN

outFileStr = ""

if not os.path.isdir(outPathRoot):
   os.makedirs(outPathRoot)

packagesJson = open(jsonPackagesfile)
packages = json.load(packagesJson)

for package in packages:

    print "interating through packages"
    print package
    print

    packageCategory = package["Category"]

    if packageCategory == '':
        #get from first featureclass in the package
        packageCategory = package["FeatureClassses"][0].split(".")[0]

    categoryFolderPath = os.path.join(outPathRoot, packageCategory)

    if not os.path.isdir(categoryFolderPath):
        os.makedirs(categoryFolderPath)

    packageFolderPath = os.path.join(outPathRoot,packageCategory, 'PackagedData','_Statewide', package["Name"])
    unpackagedFolderPath = os.path.join(outPathRoot,packageCategory,'UnpackagedData')

    if not os.path.isdir(packageFolderPath):
        os.makedirs(packageFolderPath)

    if not os.path.isdir(unpackagedFolderPath):
        os.makedirs(unpackagedFolderPath)

    print "created folders"
    print packageFolderPath
    print unpackagedFolderPath
    print

    #make geodatabase
    if arcpy.Exists(os.path.join(packageFolderPath, package["Name"] + ".gdb")):
        arcpy.Delete_management(os.path.join(packageFolderPath, package["Name"] + ".gdb"))

    arcpy.CreateFileGDB_management(packageFolderPath, package["Name"] + ".gdb", "9.3")

    print "package fgdb created"
    print

    # *** populate local packaged file geodatabase
    for fc in package["FeatureClasses"]:

        if arcpy.Exists(os.path.join(sdeConnectionStr,fc)):

            #arcpy.GetCount_management(os.path.join(sdeConnectionStr, fc ))

            #add feature class to local file geodatabase to be packaged later
            try:
                arcpy.Copy_management(os.path.join(sdeConnectionStr,fc), os.path.join(packageFolderPath, package["Name"] + ".gdb", fc))
            except:
                print 'copy _mgt exception rasied'
                arcpy.FeatureClassToFeatureClass_conversion(os.path.join(sdeConnectionStr,fc), os.path.join(packageFolderPath, package["Name"] + ".gdb"), fc.split(".")[2])

            print "fc copied to package gdb"

            #create another file gdb and copy to Unpackaged folder
            fcUnpackagedFolderPath = os.path.join(unpackagedFolderPath, fc.split(".")[2], '_Statewide')

            if not os.path.isdir(fcUnpackagedFolderPath):
                os.makedirs(fcUnpackagedFolderPath)

            print "created unpackaged statewide folder"
            print fcUnpackagedFolderPath
            print

            try:
                arcpy.CreateFileGDB_management(fcUnpackagedFolderPath, fc.split(".")[2] + ".gdb", "9.3")
            except:
                print "Error: unable to create 93 filegdb: " + os.path.join(fcUnpackagedFolderPath, fc.split(".")[2] + ".gdb")

            print "created unpackaged gdb in "
            print fcUnpackagedFolderPath
            print

            # *** populate local unpackaged file geodatabase
            arcpy.Copy_management(os.path.join(packageFolderPath, package["Name"] + ".gdb", fc.split(".")[2]), os.path.join(fcUnpackagedFolderPath, fc.split(".")[2] + ".gdb", fc.split(".")[2]))

            print "fc copied to unpackaged gdb"
            print fc
            print

            zfGDBUnpackaged = zipfile.ZipFile(os.path.join(fcUnpackagedFolderPath, fc.split(".")[2] + '_gdb.zip'), 'w', zipfile.ZIP_DEFLATED)
            zipws(os.path.join(fcUnpackagedFolderPath, fc.split(".")[2] +".gdb"), zfGDBUnpackaged, True)
            zfGDBUnpackaged.close()

            print "unpackaged gdb zipped"
            print fc
            print zfGDBUnpackaged

            arcpy.Delete_management(os.path.join(fcUnpackagedFolderPath, fc.split(".")[2]+ '.gdb'))

            #do county or other level clipping here eventually
            #if package[3] == 'SGID10.Boundaries.Counties':
            #    clipToCounties (package[3], 'SGID10.Boundaries.Counties', unpackagedFolderPath)

    print "package gdb created"


    arcpy.env.workspace = os.path.join(packageFolderPath, package["Name"] + ".gdb")

    #create zip file for shapefile package
    zfSHP = zipfile.ZipFile(os.path.join(packageFolderPath, package["Name"] + '_shp.zip'), 'w', zipfile.ZIP_DEFLATED)
    arcpy.env.overwriteOutput = True #Overwrite pre-existing files


    #output zipped shapefiles for each feature class
    fileGDB_FCs = arcpy.ListFeatureClasses()


    for fc in fileGDB_FCs:

        print "Starting " + fc

        #create shapefile for the feature class
        try:
            arcpy.FeatureClassToShapefile_conversion(os.path.join(packageFolderPath, package["Name"]+".gdb", fc), packageFolderPath)
            print "Exported " + os.path.join(packageFolderPath, package["Name"]+".gdb", fc) + " to shapefile at" + packageFolderPath
        except:
            print "Could not export " + os.path.join(packageFolderPath, package["Name"]+".gdb", fc) + " to shapefile at" + packageFolderPath

        #add to package zipfile package
        zipws(packageFolderPath, zfSHP, False)

        #create unpackaged zip file and move data into that zip file
        zfSHPUnpackaged = zipfile.ZipFile(os.path.join(unpackagedFolderPath, fc, '_Statewide', fc + '_shp.zip'), 'w', zipfile.ZIP_DEFLATED)
        zipws(packageFolderPath, zfSHPUnpackaged, False)
        zfSHPUnpackaged.close()

        # add to .bat file str for unpackaged data
        outFileStr = outFileStr + "del " + os.path.join(ftpPathRoot, packageCategory, 'UnpackagedData', fc, "_Statewide","*.*") + " /q\n"
        outFileStr = outFileStr + "rmdir " + os.path.join(ftpPathRoot, packageCategory,'UnpackagedData', fc, "_Statewide") + " /q\n"
        outFileStr = outFileStr + "rmdir " + os.path.join(ftpPathRoot, packageCategory,'UnpackagedData', fc) + " /q\n"
        outFileStr = outFileStr + "mkdir " + os.path.join(ftpPathRoot, packageCategory,'UnpackagedData', fc) + "\n"
        outFileStr = outFileStr + "mkdir " + os.path.join(ftpPathRoot, packageCategory,'UnpackagedData', fc, '_Statewide') + "\n"

        outFileStr = outFileStr + "copy " + (os.path.join(unpackagedFolderPath, fc, '_Statewide', fc + '_gdb.zip')) + " " + (os.path.join(ftpPathRoot,packageCategory,'UnpackagedData',fc,'_Statewide', fc + '_gdb.zip')) + "\n"
        outFileStr = outFileStr + "copy " + (os.path.join(unpackagedFolderPath, fc, '_Statewide', fc + '_shp.zip')) + " " + (os.path.join(ftpPathRoot,packageCategory,'UnpackagedData',fc,'_Statewide', fc + '_shp.zip')) + "\n"

        #delete temporary shapefiles
        try:
            arcpy.Delete_management(os.path.join(packageFolderPath, fc + ".shp"))
        except:
            print "Could not delete " + os.path.join(packageFolderPath, fc + ".shp")

    zfSHP.close()


    #zip package geodatabase
    outFileStr = outFileStr + "\n"
    outFileStr = outFileStr + "REM update PACKAGED GDB files" + "\n"

    zfFGDB = zipfile.ZipFile(os.path.join(packageFolderPath, package["Name"] + '_gdb.zip'), 'w', zipfile.ZIP_DEFLATED)
    target_dir = os.path.join(packageFolderPath, package["Name"]+'.gdb')
    rootlen = len(target_dir) + 1
    for base, dirs, files in os.walk(target_dir):
       for file in files:
          fn = os.path.join(base, file)
          zfFGDB.write(fn, package["Name"]+".gdb/" + fn[rootlen:])
    zfFGDB.close()

    # add to .bat file str for packages
    outFileStr = outFileStr + "del " + os.path.join(ftpPathRoot, packageCategory, 'PackagedData', '_Statewide', package["Name"],"*.*") + " /q\n"
    outFileStr = outFileStr + "rmdir " + os.path.join(ftpPathRoot, packageCategory, 'PackagedData', '_Statewide', package["Name"]) + " /q\n"
    outFileStr = outFileStr + "mkdir " + os.path.join(ftpPathRoot, packageCategory,'PackagedData','_Statewide', package["Name"]) + "\n"
    outFileStr = outFileStr + "copy " + (os.path.join(packageFolderPath, package["Name"] + '_gdb.zip')) + ' ' + (os.path.join(ftpPathRoot,packageCategory,'PackagedData','_Statewide',package["Name"],package["Name"] + '_gdb.zip')) + "\n"
    outFileStr = outFileStr + "copy " + (os.path.join(packageFolderPath, package["Name"] + '_shp.zip')) + ' ' + (os.path.join(ftpPathRoot,packageCategory,'PackagedData','_Statewide',package["Name"],package["Name"] + '_shp.zip')) + "\n"
    outFileStr = outFileStr  + ('\n')

    #delete temp geodatabase
    arcpy.Delete_management(os.path.join(packageFolderPath,package["Name"]+".gdb"))

    outfile = open(os.path.join(outPathRoot,"movesgidout_" + package["Name"] + "_" + strftime("%Y%m%d%H%M%S")  + ".bat"), "w")

    # write .bat file
    outfile.write('\n')
    outfile.write('\n')
    outfile.write('REM Started: ' + strftime("%Y-%m-%d %H:%M:%S")  + "\n")
    outfile.write('REM Finished: ' + strftime("%Y-%m-%d %H:%M:%S") + "\n")
    outfile.write(outFileStr + '\n')
    outfile.close()

print "Completed"