Skip to content

Instantly share code, notes, and snippets.

@tormi
Forked from thomasneirynck/geojson_to_index.js
Created November 2, 2020 12:30
Show Gist options
  • Save tormi/7e1286d2c9697180014d5035d16507de to your computer and use it in GitHub Desktop.
Save tormi/7e1286d2c9697180014d5035d16507de to your computer and use it in GitHub Desktop.
Ingest geojson file into Elasticsearch index. Each feature in the FeatureCollection corresponds to a document in Elasticsearch.
const fs = require("fs");
const elasticsearch = require('elasticsearch');
const oboe = require('oboe');
const geojsonInput = process.argv[2] || 'feature_collection.geojson';
const indexname = process.argv[3] || geojsonInput.split('.')[0] || 'feature_collection';
const geometryFieldName = 'geometry';
const shape_type = process.argv[4] || 'geo_shape';
if (shape_type !== 'geo_point' && shape_type !== 'geo_shape') {
console.error(`Invalid shapetype ${shape_type}`);
return;
}
async function ingest() {
const esClient = new elasticsearch.Client({
host: 'elastic:changeme@localhost:9200',
version: '7.1',
// log: 'trace'
});
try {
await esClient.ping({
requestTimeout: 1000
});
} catch (e) {
console.error('Cannot reach Elasticsearch', e);
throw e;
}
try {
await esClient.indices.delete({
index: indexname
});
} catch (e) {
console.warn(e);
}
try {
console.log('shape type', shape_type);
await esClient.indices.create({
index: indexname,
body: {
mappings: {
"properties": {
[geometryFieldName]: {
"type": shape_type,
"ignore_malformed": true
}
}
}
}
});
} catch (e) {
console.error(e);
throw e;
}
const readStream = fs.createReadStream(geojsonInput);
let i = 0;
oboe(readStream)
.node('features.*',async (feature) => {
const geometry = shape_type === 'geo_point' ? feature.geometry.coordinates : feature.geometry;
const doc = {
...feature.properties,
[geometryFieldName]: geometry
};
try {
await esClient.create({
id: i++,
index: indexname,
// type: '_doc', //ES 80 does not use types
body: doc
});
console.log(`Created ${i}`);
} catch (e) {
console.error(e);
throw e;
}
})
.done(()=>{
console.log('done processing');
});
}
ingest();
@tormi
Copy link
Author

tormi commented Nov 2, 2020

Requires elasticsearch and oboe dependency.

npm install elasticsearch
npm install oboe

Example command: > node ./geojson_to_index.js boroughs.json boroughs geo_shape

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment