GeoJsonItemExporter for Scrapy

README.md

About the GeoJsonItemExporter for Scrapy

The GeoJsoItemExporter creates a collection of features where the feature's geometry property is a point and its properties property is the item serialized into JSON format.

Every item must have a value for latitude and longitude in order to be exported. No validation is made to ensure the data is correct.

exporters.py

# Based on Scrapy's JsonItemExporter
# See: https://github.com/scrapy/scrapy/blob/master/scrapy/exporters.py
from scrapy.exporters import BaseItemExporter
from scrapy.utils.serialize import ScrapyJSONEncoder
from scrapy.utils.python import to_bytes

class GeoJsonItemExporter(BaseItemExporter):

    def __init__(self, file, **kwargs):
        self._configure(kwargs, dont_fail=True)
        self.file = file
        kwargs.setdefault('ensure_ascii', not self.encoding)
        self.encoder = ScrapyJSONEncoder(**kwargs)
        self.first_item = True

    def start_exporting(self):
        self.file.write(b'{ "type": "FeatureCollection","features":[\n')

    def finish_exporting(self):
        self.file.write(b"\n]}")

    def export_item(self, item):
        if self.first_item:
            self.first_item = False
        else:
            self.file.write(b',\n')
        itemdict = dict(self._get_serialized_fields(item))
        geojsondict = {
            'type':'Feature',
            'geometry': {
                'type': 'Point',
                'coordinates': [itemdict['longitude'], itemdict['latitude']]
            },
            'properties':itemdict
        }
        data = self.encoder.encode(geojsondict)
        self.file.write(to_bytes(data, self.encoding))

pipelines.py

from crawler.exporters import GeoJsonItemExporter

class GeoJSONWriterPipeline(object):

    def __init__(self):
        self.files = dict()
        self.exporters = dict()

    def open_spider(self, spider):
        file_path = '/data/{}.geojson'.format(spider.name)
        file = open(file_path, spider._meta.get('mode', 'wb'))
        self.files[spider.name] = file
        self.exporters[spider.name] = GeoJsonItemExporter(file)
        self.exporters[spider.name].start_exporting()

    def process_item(self, item, spider):
        if item.get('latitude') and item.get('longitude'):
            self.exporters[spider.name].export_item(item)
        return item

    def close_spider(self, spider):
        exporter = self.exporters.pop(spider.name)
        exporter.finish_exporting()
        file = self.files.pop(spider.name)
        file.close()