Helper function to harvest all records of an ArcGIS feature service

harvest_feature_service.py

import arcpy
import urllib
import json


def harvest_feature_service(baseURL, whereClause="", fields="*", token="", messages=None):
    """
    Return all records from a feature service as a feature set
    :param baseURL: The base URL of the feature service, including the layer ID
    :param whereClause: SQL where clause (optional)
    :param fields: Comma separate list for field to return (optional)
    :param token: Token for authentication (optional)
    :param messages: Message object for use in Python Toolbox (optional)
    :return: arcpy.FeatureSet
    """

    # Get ObjectID field name and max record count
    response = json.loads(urllib.urlopen("{}?token={}f=pjson".format(baseURL, token)).read())
    if "type" in response.keys():
        if response['type'] == "Feature Layer":
            id_field = None
            if "objectIdField" in response.keys():
                id_field = response["objectIdField"]
            else:
                for field in response['fields']:
                    if field['type'] == 'esriFieldTypeOID':
                        id_field = field['name']
            max_record_count = response["maxRecordCount"]

            if id_field is None:
                raise Exception("Cannot identify object ID field")
        else:
            raise Exception("Not a feature layer")
    else:
        raise Exception("Not a feature layer")

    # Order by ObjectID
    orderByFields = id_field

    # initiate counters
    max_id = -1
    record_count = 0
    cycle = 0
    records_left = max_record_count + 1

    #initatiate output feature set
    output_fs = arcpy.FeatureSet()

    output = "Fetch rows"
    if messages is not None:
        messages.addMessage(output)
    else:
        print output

    while records_left > max_record_count:

        # construct the where clause
        if whereClause == "":
            where = '{}>{}'.format(id_field, max_id)
        else:
            where = '({}) AND {}>{}'.format(whereClause, id_field, max_id)

        # construct count query
        count_query = "/query?where={}&returnCountOnly=true&f=json&orderByFields={}&token={}".format(where, orderByFields, token)
        countURL = baseURL + count_query

        # construct the query
        query = "/query?where={}&outFields={}&returnGeometry=true&f=json&orderByFields={}&token={}".format(where, fields, orderByFields, token)
        fsURL = baseURL + query

        # count remaining records
        response = json.loads(urllib.urlopen(countURL).read())
        records_left = response["count"]
        if cycle == 0:
            record_count = records_left

        # fetch data
        fs = arcpy.FeatureSet()
        fs.load(fsURL)

        if max_record_count < records_left:
            records_return = max_record_count
        else:
            records_return = records_left

        output = "{} records retrieved, {} records left".format(records_return, records_left-records_return)
        if messages is not None:
            messages.addMessage(output)
        else:
            print output

        # check max ObjectID
        cur = arcpy.SearchCursor(fs)
        for row in cur:
            id = row.getValue(id_field)
            if id > max_id:
                max_id = id

        # append results to output feature set
        if cycle == 0:
            output_fs.load(fs)
        else:
            arcpy.Append_management(fs, output_fs)

        cycle += 1

    # confirm record number
    output = "{} rows returned".format(record_count)
    if messages is not None:
        messages.addMessage(output)
    else:
        print output

    # return results
    return output_fs

Use like this

result = harvest_feature_service("http://URL/to/FeatureLayer", token='XYZ')

# check number of returned records
cur = arcpy.SearchCursor(result)
i = 0
for row in cur:
    i += 1
print i

# save as feature class
arcpy.CopyFeatures_management(result, r"c:\local\data.gdb\my_records")