Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
#-------------------------------------------------------------------------------
# Name: UploadPublishCSVMultipart
# Purpose: This script uploads and publishes a CSV that includes XY data, not
# addresses. This uses a multipart upload for the CSV and requires
# username, password, CSVfile, X and Y fields, and tags to be defined
# Author: Melanie Summers
import requests
import os
import json
def generateToken(username, password):
'''Generate a token using the requests module for the input
username and password'''
url = "https://arcgis.com/sharing/generateToken"
data = {'username': username,
'password': password,
'referer' : 'https://arcgis.com',
'expiration' : 1209600,
'f': 'json'}
request = requests.post(url, data=data, verify=False)
return request.json()['token']
def GetInfo(token):
'''Get information about the specified organization
this information includes the short name of the organization (['urlKey'])
as well as the organization ID ['id']'''
URL= 'https://arcgis.com/sharing/rest/portals/self?f=json&token={}'.format(token)
response = requests.get(URL, verify=False)
return response.json()
def uploadItem(short, username, token, itemName):
'''Upload the input CSV, this is using a post request through the requests module,
to access the itemID, use the index ['id']'''
uploadURL = 'http://{}.maps.arcgis.com/sharing/rest/content/users/{}/addItem?token={}&f=json'.format(short, username, token)
data = {'multipart': 'true',
'filename' : itemName,
'type': 'CSV'}
response = requests.post(uploadURL, data=data)
return response.json()
def SplitFile(inputfile):
'''Splits the input file into smaller portions to be added individually
returns the temporary folder location where the files are stored.'''
Folder = tempfile.mkdtemp()
opened = open(inputfile, 'r')
totalsize = os.path.getsize(inputfile)
midsize = totalsize/6291000
if midsize > 0:
for X in range(1, midsize +1):
lines = opened.read(totalsize/midsize)
interfile = open(os.path.join(Folder, 'Data' + str(X) + '.csv'), 'w')
interfile.write(lines)
interfile.close()
opened.close()
else:
lines = opened.read()
interfile = open(os.path.join(Folder, 'Data1.csv'), 'w')
interfile.write(str(lines))
interfile.close()
opened.close()
return Folder
def addPart(short, username, CSVID, token, filename, num):
'''Add part of the input CSV using a post request through the request module'''
addpartURL = 'http://{}.maps.arcgis.com/sharing/rest/content/users/{}/items/{}/addPart?token={}&f=json'.format(short, username, CSVID, token)
data = {'file': open(filename, 'rb')}
url = addpartURL + "&file="+filename+ "&partNum="+str(num)
response = requests.post(url, files = data)
return json.loads(response.text)
def StatusTest(TYPE, short, username, ItemID, token, jobID=''):
'''return the job status, used to confirm the item was uploaded
or the service published successfully before proceeding'''
if TYPE == 'CSV':
TestURL = "http://{}.maps.arcgis.com/sharing/rest/content/users/{}/items/{}/status?f=json&token={}".format(short, username, ItemID, token)
elif TYPE == 'SERVICE':
TestURL = "http://{}.maps.arcgis.com/sharing/rest/content/users/{}/items/{}/status?jobid={}&f=json&token={}".format(short, username, ItemID, jobID, token)
result = requests.get(TestURL)
return result.json()
def commit(short, username, ID, token):
'''Used as part of the multipart upload to combine the parts,
the order is defined by the partnumber specified when
each part was added'''
commitURL = "http://{}.maps.arcgis.com/sharing/rest/content/users/{}/items/{}/commit?f=json&token={}".format(short, username, ID, token)
response = requests.post(commitURL)
return json.loads(response.text)
def updateItem(short, username, ID, token, itemTitle, tags, filename):
'''Updates the item details title, tags, filename, type, and typekeywords'''
updateURL = "http://{}.maps.arcgis.com/sharing/content/users/{}/items/{}/update?f=json&token={}".format(short, username, ID, token)
data = {'title': itemTitle,
'tags': tags,
'filename': filename,
'typeKeywords': 'CSV',
'type': 'CSV'}
response = requests.post(updateURL, data=data)
return json.loads(response.text)
def analyzeCSV(token, CSVID, FileName):
'''Analyzes the input CSV to generate the JSON used
when publishing the CSV'''
url = 'http://www.arcgis.com/sharing/rest/content/features/analyze?token={}'.format(token)
data = {'f': 'JSON',
'itemid': CSVID,
'file': FileName,
'filetype':'csv'}
request = requests.post(url, data=data)
return request.json()
def decode_list(feature):
rv=[]
for item in feature:
if isinstance(item, unicode):
item = item.encode('utf-8')
elif isinstance(item, list):
item = decode_list(item)
elif isinstance(item, dict):
item = decode_dict(item)
rv.append(item)
return rv
def decode_dict(data):
rv = {}
for key, value in data.iteritems():
if isinstance(key, unicode):
key = key.encode('utf-8')
if isinstance(value, unicode):
value = value.encode('utf-8')
elif isinstance(value, list):
value = decode_list(value)
elif isinstance(value, dict):
value = decode_dict(value)
rv[key] = value
return rv
def PublishService(Short, itemID, username, token, itemName, publishParams, XField, YField):
''' Publishes the input itemID (uploaded CSV) this is using
a post request from urllib, the JSON input can be generated by
the analyze function. Item name, X and Y fields are specified'''
publishURL = 'http://{}.maps.arcgis.com/sharing/rest/content/users/{}/publish'.format(Short, username)
publishParams['name'] = itemName
publishParams['locationType'] = 'coordinates'
publishParams['latitudeFieldName'] = YField
publishParams['longitudeFieldName'] = XField
query_dict = {
'itemID': itemID,
'filetype': 'csv',
'f': 'json',
'token': token,
'publishParameters':publishParams}
query_dict['publishParameters'] = json.dumps(query_dict['publishParameters'], sort_keys=False)
request = requests.post(publishURL, data=query_dict)
return request.json()
def TestQuery(ServiceURL, token):
'''This is a test query to ensure the service was published succcessfully'''
QueryURL = "{}/0/query?where=1=1&returnCountOnly=True&f=json&token={}".format(ServiceURL, token)
result = requests.get(QueryURL)
return result.json()
if __name__ == '__main__':
username = raw_input("Please enter your username: ")
password = raw_input("Please enter your password: ")
ItemName = raw_input("Enter the name of the service")
Tags = 'tags'
CSV = raw_input("Please enter the path to the CSV file: ")
XField = raw_input("What is the X/Latitude field name? ")
YField = raw_input("What is the Y/Longitude field name? ")
token = generateToken(username, password)
Info = GetInfo(token)
Short = Info['urlKey']
ItemID = uploadItem(Short, username, token, ItemName)['id']
TempDir = SplitFile(CSV)
Count = 0
for a, b, c in os.walk(TempDir):
for name in c:
print name
index = name.replace('Data', '').replace('.csv', '')
PART = addPart(Short, username, ItemID, token, os.path.join(TempDir,name), index)
commit(Short, username, ItemID, token)
while StatusTest('CSV', short, username, ItemID, token)['status']!='completed':
if StatusTest('CSV', short, username, ItemID, token)['status'] == 'failed':
print "CSV upload failed"
break
time.sleep(5)
print ("Processing CSV")
updateItem(Short, username, ItemID, token, ItemName, Tags, os.path.basename(CSV))
Analyzed = analyzeCSV(token, ItemID, os.path.basename(CSV))
decoded = decode_dict(Analyzed['publishParameters'])
OutputURL = PublishService(Short, ItemID, username, token, ItemName, decoded, XField, YField)
while StatusTest('SERVICE', Short, username, OutputURL['services'][0]['serviceItemId'], token,OutputURL['services'][0]['jobId'])['status']!='completed':
if StatusTest('SERVICE',Short, username, OutputURL['services'][0]['serviceItemId'], token,OutputURL['services'][0]['jobId'])['status'] == 'failed':
print"Publishing service failed"
break
time.sleep(25)
print("Processing Service")
print TestQuery(OutputURL['services'][0]['serviceurl'], token)['count']
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.