Last active
September 19, 2017 18:11
-
-
Save thomas-maschler/493a32c41a56fb7f7a3aa6a98bf98ec6 to your computer and use it in GitHub Desktop.
Helper function to harvest all records of an ArcGIS feature service
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import arcpy | |
import urllib | |
import json | |
def harvest_feature_service(baseURL, whereClause="", fields="*", token="", messages=None): | |
""" | |
Return all records from a feature service as a feature set | |
:param baseURL: The base URL of the feature service, including the layer ID | |
:param whereClause: SQL where clause (optional) | |
:param fields: Comma separate list for field to return (optional) | |
:param token: Token for authentication (optional) | |
:param messages: Message object for use in Python Toolbox (optional) | |
:return: arcpy.FeatureSet | |
""" | |
# Get ObjectID field name and max record count | |
response = json.loads(urllib.urlopen("{}?token={}f=pjson".format(baseURL, token)).read()) | |
if "type" in response.keys(): | |
if response['type'] == "Feature Layer": | |
id_field = None | |
if "objectIdField" in response.keys(): | |
id_field = response["objectIdField"] | |
else: | |
for field in response['fields']: | |
if field['type'] == 'esriFieldTypeOID': | |
id_field = field['name'] | |
max_record_count = response["maxRecordCount"] | |
if id_field is None: | |
raise Exception("Cannot identify object ID field") | |
else: | |
raise Exception("Not a feature layer") | |
else: | |
raise Exception("Not a feature layer") | |
# Order by ObjectID | |
orderByFields = id_field | |
# initiate counters | |
max_id = -1 | |
record_count = 0 | |
cycle = 0 | |
records_left = max_record_count + 1 | |
#initatiate output feature set | |
output_fs = arcpy.FeatureSet() | |
output = "Fetch rows" | |
if messages is not None: | |
messages.addMessage(output) | |
else: | |
print output | |
while records_left > max_record_count: | |
# construct the where clause | |
if whereClause == "": | |
where = '{}>{}'.format(id_field, max_id) | |
else: | |
where = '({}) AND {}>{}'.format(whereClause, id_field, max_id) | |
# construct count query | |
count_query = "/query?where={}&returnCountOnly=true&f=json&orderByFields={}&token={}".format(where, orderByFields, token) | |
countURL = baseURL + count_query | |
# construct the query | |
query = "/query?where={}&outFields={}&returnGeometry=true&f=json&orderByFields={}&token={}".format(where, fields, orderByFields, token) | |
fsURL = baseURL + query | |
# count remaining records | |
response = json.loads(urllib.urlopen(countURL).read()) | |
records_left = response["count"] | |
if cycle == 0: | |
record_count = records_left | |
# fetch data | |
fs = arcpy.FeatureSet() | |
fs.load(fsURL) | |
if max_record_count < records_left: | |
records_return = max_record_count | |
else: | |
records_return = records_left | |
output = "{} records retrieved, {} records left".format(records_return, records_left-records_return) | |
if messages is not None: | |
messages.addMessage(output) | |
else: | |
print output | |
# check max ObjectID | |
cur = arcpy.SearchCursor(fs) | |
for row in cur: | |
id = row.getValue(id_field) | |
if id > max_id: | |
max_id = id | |
# append results to output feature set | |
if cycle == 0: | |
output_fs.load(fs) | |
else: | |
arcpy.Append_management(fs, output_fs) | |
cycle += 1 | |
# confirm record number | |
output = "{} rows returned".format(record_count) | |
if messages is not None: | |
messages.addMessage(output) | |
else: | |
print output | |
# return results | |
return output_fs |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Use like this