Skip to content

Instantly share code, notes, and snippets.

@roman-yepishev
Last active August 1, 2016 02:28
Show Gist options
  • Save roman-yepishev/a9f0f3b51333772d7c80713bce24c866 to your computer and use it in GitHub Desktop.
Save roman-yepishev/a9f0f3b51333772d7c80713bce24c866 to your computer and use it in GitHub Desktop.
Brute-force ArcGIS Feature Layer to GeoJSON exporter
#!/usr/bin/python3
import json
import os
import sys
import math
import requests
import fiona
import argparse
from collections import OrderedDict
REQUEST_SIZE=1000
GEOMETRY_TYPE_MAP = {
'esriGeometryPoint': 'Point',
'esriGeometryPolygon': 'Polygon',
'esriGeometryPolyline': 'MultiLineString'
}
ATTRIBUTE_TYPE_MAP = {
'esriFieldTypeInteger': 'int',
'esriFieldTypeString': 'str',
'esriFieldTypeDouble': 'float',
'esriFieldTypeDate': 'str'
}
class Converter(object):
def __init__(self):
self._features = {}
self._schema = {
'properties': OrderedDict({})
}
def _add(self, feature):
object_id = feature['id']
if object_id not in self._features:
self._features[object_id] = feature
def parse(self, fs_json):
if not self._schema['properties']:
if 'geometryType' in fs_json:
self._schema['geometry'] = GEOMETRY_TYPE_MAP[fs_json['geometryType']]
if 'fields' not in fs_json:
return
for item in fs_json['fields']:
print(item)
if item['type'] in ATTRIBUTE_TYPE_MAP:
type_ = ATTRIBUTE_TYPE_MAP[item['type']]
else:
type_ = 'str'
if 'length' in item:
type_ += ':' + str(item['length'])
self._schema['properties'][item['name']] = type_
if 'features' not in fs_json:
return
for item in fs_json['features']:
item_geometry = item['geometry']
if self._schema['geometry'] == 'Point':
coordinates = [item_geometry['x'], item_geometry['y']]
elif self._schema['geometry'] == 'Polygon':
coordinates = item['geometry']['rings']
elif self._schema['geometry'] == 'MultiLineString':
coordinates = item['geometry']['paths']
for id_attr in ['OBJECTID', 'OID']:
if id_attr in item['attributes']:
id_ = item['attributes'][id_attr]
break
feature = {
'geometry': {
'type': self._schema['geometry'],
'coordinates': coordinates
},
'properties': {
str(k): str(v) for k,v in item['attributes'].items()
},
'id': id_
}
self._add(feature)
def serialize(self, output):
print(self._schema)
with fiona.open(output, 'w', schema=self._schema,
driver='GeoJSON') as sink:
for feature in self._features.values():
sink.write(feature)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description='Convert FeatureServer JSON to GeoJSON'
)
parser.add_argument('url', type=str,
help='FeatureServer endpoint URL')
parser.add_argument('--layer', '-l', dest='layer', type=str,
default='0', help='FeatureServer layer id or name')
parser.add_argument('output', type=str,
help='Output GeoJSON file')
parser.add_argument('--force', '-f', dest='force', action='store_true',
default=False, help='Force overwriting the file')
parser.add_argument('--size', '-s', dest='request_size',
default=REQUEST_SIZE, type=int,
help='Set request size for X and Y')
parser.add_argument('--one', '-o', dest='one',
default=False,
action='store_true',
help='Run conversion on the first batch of data only')
args = parser.parse_args()
if os.path.exists(args.output) and not args.force:
print("Output file {} exists, will not overwrite without -f".format(
args.output))
sys.exit(1)
params = {
'f': 'json'
}
response = requests.get(args.url, params)
metadata = response.json()
layer_id = None
for layer in metadata['layers']:
if layer['name'] == args.layer or str(layer['id']) == args.layer:
layer_id = layer['id']
if layer_id is None:
print("Layer '{}' was not found in metadata".format(args.layer))
print('Available layers:')
for layer in metadata['layers']:
print('\t{}: {}'.format(layer['id'], layer['name']))
sys.exit(1)
url = '{}/{}'.format(args.url, layer_id)
response = requests.get(url, params)
layer_metadata = response.json()
extent = layer_metadata['extent']
ranges = {
'y': range(
math.floor(extent['ymin']),
math.ceil(extent['ymax']),
args.request_size
),
'x': range(
math.floor(extent['xmin']),
math.ceil(extent['xmax']),
args.request_size
)
}
c = Converter()
url = '{}/{}/query'.format(args.url, layer_id)
params = {
'f': 'json',
'geometry': '',
'geometryType': 'esriGeometryEnvelope',
'inSR': extent['spatialReference']['wkid'],
'spatialRel': 'esriSpatialRelIntersects',
'returnGeometry': 'true',
'outFields':'*',
'outSR': 4326
}
total = len(ranges['y']) * len(ranges['x'])
counter = 0
print("Ranges:\tx={}\n\ty={}".format(ranges['x'], ranges['y']))
geometry = {
"xmin": 0,
"ymin": 0,
"xmax": 0,
"ymax": 0,
"spatialReference": extent['spatialReference']
}
overlap = args.request_size / 100 * 10
for y in ranges['y']:
geometry['ymin'] = y - overlap
geometry['ymax'] = y + overlap + args.request_size
for x in ranges['x']:
geometry['xmin'] = x - overlap
geometry['xmax'] = x + overlap + args.request_size
params['geometry'] = json.dumps(geometry)
counter += 1
print('Fetching data... {:8.2f}% ({}/{})'.format(
counter / total * 100,
counter, total))
response = requests.get(url, params)
c.parse(response.json())
if args.one:
break
if args.one:
break
if os.path.exists(args.output):
os.unlink(args.output)
c.serialize(args.output)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment