Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save florinvirdol/f09ba297c63d2211c94964d3160a7ea5 to your computer and use it in GitHub Desktop.
Save florinvirdol/f09ba297c63d2211c94964d3160a7ea5 to your computer and use it in GitHub Desktop.
/*
Mostly copied and adapted from RussCam's GIST: https://gist.github.com/russcam/5c40e5c6fb95b9afd9242df675fce428
*/
/*
Questions:
1. Regarding not working scenarios (#3, #4, #5) how can i deserialize a GeoJson string into a Geometry object?
2. Regarding Scenario #6, why GeometryCollection data isn't visibile on Kibana Map like simple Geometry (Polygons)?
- 2.1. Don't know if it's related, but when dragging on Kibana Maps i get this error:
`Error {message: "Input data given to 'cfe5e9a5-de63-4beb-85b2-4b67ad455ae9' is not a valid GeoJSON object."}
message: "Input data given to 'cfe5e9a5-de63-4beb-85b2-4b67ad455ae9' is not a valid GeoJSON object."_ proto _: Object
overrideMethod @ react_devtools_backend.js:2430
Ut.fire @ maps.chunk.1.js:31
3. What are the differences and pro / cons of ingesting ElasticSearch geoshape as multiple individual Geometry objects (polygons in individual documents) rather than one single GeometryCollection (polygons in a single document)?
- 3.1. What about indexing (execution time and performance - even when using Bulk Index)?
- 3.2. What about querying, searching, filtering (usability, performance, etc.)?
4. Regarding Scenario #7, why some Geometries seems invalid?
- 4.1. Does NTS have a MakeValid method? Or how can i fix this in C#?
*/
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection.Metadata;
using System.Text;
using Bogus.DataSets;
using Elasticsearch.Net;
using ElasticSearch;
using GeoAPI.Geometries;
using Microsoft.Extensions.Configuration;
using Nest;
using Nest.JsonNetSerializer;
using NetTopologySuite.Features;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NetTopologySuite.IO;
using NetTopologySuite.Geometries;
using NetTopologySuite.IO.Converters;
using Newtonsoft.Json.Converters;
using Coordinate = NetTopologySuite.Geometries.Coordinate;
using GeometryCollection = NetTopologySuite.Geometries.GeometryCollection;
private static void Main()
{
try {
var defaultIndex = "my_shapes";
string cloudId = "cloudId";
string username = "username";
string password = "password";
var credentials = new BasicAuthenticationCredentials(username, password);
//var pool = new SingleNodeConnectionPool(new Uri($"http://localhost:9200"));
var pool = new CloudConnectionPool(cloudId, credentials);
var settings = new ConnectionSettings(pool, (c, s) =>
new JsonNetSerializer(c, s, contractJsonConverters: new JsonConverter[]
{
new AttributesTableConverter(),
new CoordinateConverter(),
new EnvelopeConverter(),
new FeatureConverter(),
new FeatureCollectionConverter(),
new GeometryConverter(),
new GeometryArrayConverter(),
new StringEnumConverter()
}))
.DefaultIndex(defaultIndex)
.DisableDirectStreaming()
.PrettyJson()
.OnRequestCompleted(callDetails => {
if (callDetails.RequestBodyInBytes != null) {
var json = JObject.Parse(Encoding.UTF8.GetString(callDetails.RequestBodyInBytes));
Console.WriteLine(
$"{callDetails.HttpMethod} {callDetails.Uri} \n" +
$"{json.ToString(Newtonsoft.Json.Formatting.Indented)}");
}
else {
Console.WriteLine($"{callDetails.HttpMethod} {callDetails.Uri}");
}
Console.WriteLine();
if (callDetails.ResponseBodyInBytes != null) {
Console.WriteLine($"Status: {callDetails.HttpStatusCode}\n" +
$"{Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}\n" +
$"{new string('-', 30)}\n");
}
else {
Console.WriteLine($"Status: {callDetails.HttpStatusCode}\n" +
$"{new string('-', 30)}\n");
}
});
var client = new ElasticClient(settings);
var createIndexResponse = client.Indices.Create(defaultIndex, c => c
.Map<MyDocument>(m => m
.Properties(p => p
.GeoShape(g => g
.Name(n => n.Geometry)
)
)
)
);
if (!createIndexResponse.IsValid) {
throw new Exception($"Error creating index: {createIndexResponse.DebugInformation}");
}
IndexResponse indexResponse;
MyDocument document;
Geometry geometryPolygon;
FeatureCollection featureCollection;
//Working Scenario #1: Geometry from mock Polygon -------------------works!!!!!!!!!!!
var polygon = new Polygon(new LinearRing(new [] {
new Coordinate(0, 0),
new Coordinate(0, 4),
new Coordinate(4, 4),
new Coordinate(4, 0),
new Coordinate(0, 0)
}));
document = new MyDocument(1, polygon);
indexResponse = client.IndexDocument(document);
//End of Scenario #1 -------------------
//Working Scenario #2: Geometry from FeatureCollection from real GeoJson file ------------------- works
var geojsonFileName = @"..\..\..\_GeoDataFiles\GeoJSONs\PostalArea.geojson";
var jsonData = File.ReadAllText(geojsonFileName);
featureCollection = new GeoJsonReader().Read<FeatureCollection>(jsonData);
if (featureCollection == null) return;
var geometry = featureCollection[0].Geometry;
document = new MyDocument(1, geometry);
indexResponse = client.IndexDocument(document);
//End of Scenario #2-------------------
//NOT Working Scenario #3: Geometry deserialized (with GeoJsonSerializer) from mock GeoJson string -------------------
//excluded coordinates arrays for clarity
var geoJsonPolygonStr1 = "{\"type\":\"Polygon\",\"coordinates\":[ ... ]}";
var serializer = new NetTopologySuite.IO.GeoJsonSerializer();
using(var stringReader = new StringReader(geoJsonPolygonStr1))
using (var jsonReader = new JsonTextReader(stringReader))
{
/*Error:
{"Could not create an instance of type NetTopologySuite.Geometries.Geometry.
Type is an interface or abstract class and cannot be instantiated.
Path 'type', line 2, position 8."}*/
geometryPolygon = serializer.Deserialize<Geometry>(jsonReader);
}
document = new MyDocument(1, geometryPolygon);
indexResponse = client.IndexDocument(document);
//End of Scenario #3 -------------------
//NOT Working Scenario #4: Geometry deserialized (with JsonConvert) from mock GeoJson string -------------------
//excluded coordinates arrays for clarity
var geoJsonPolygonStr2 = "{\"type\":\"Polygon\",\"coordinates\":[ ... ]}";
/*Error:
{"Could not create an instance of type NetTopologySuite.Geometries.Geometry.
Type is an interface or abstract class and cannot be instantiated.
Path 'type', line 2, position 8."}*/
geometryPolygon = JsonConvert.DeserializeObject<Geometry>(geoJsonPolygonStr2);
document = new MyDocument(1, geometryPolygon);
indexResponse = client.IndexDocument(document);
//End of Scenario #4 -------------------
//NOT Working Scenario #5: GeometryCollection deserialized (with JsonConvert) from mock GeoJson string -------------------
var geoCollectionMock =
@"{""type"": ""geometrycollection"",
""geometries"": ["
+ geoJsonPolygonStr1 +
","
+ geoJsonPolygonStr2 +
@"]
}";
/*Error:
{"Could not create an instance of type NetTopologySuite.Geometries.Geometry.
Type is an interface or abstract class and cannot be instantiated.
Path 'type', line 2, position 8."}*/
geometryPolygon = JsonConvert.DeserializeObject<Geometry>(geoCollectionMock);
document = new MyDocument(1, geometryPolygon);
indexResponse = client.IndexDocument(document);
//End of Scenario #5 -------------------
//Weired Scenario #6: GeometryCollection built from multiple Geometry objects from FeatureCollection from real GeoJson file -------------------
//Data ingested into ElasticSearch Index, BUT, polygons from GeometryCollection can't be seen on Kibana Maps as other simple Polygons can be seen
var geoCollectionObj = new NetTopologySuite.Geometries.GeometryCollection(new[]
{
featureCollection[0].Geometry,
featureCollection[1].Geometry,
featureCollection[2].Geometry
});
document = new MyDocument(1, geoCollectionObj);
indexResponse = client.IndexDocument(document);
//End of Scenario #6 -------------------
//Not working Scenario #7: Geometry from FeatureCollection from real GeoJson file - invalid Geometry -------------------
var isValid = featureCollection[0].Geometry.IsValid;//= false
/*Error:
"type" : "mapper_parsing_exception",
"reason" : "failed to parse field [geometry] of type [geo_shape]",
"caused_by" : {
"type" : "invalid_shape_exception",
"reason" : "Self-intersection at or near point [-3.173,57.545]"
}*/
document = new MyDocument(99, featureCollection[99].Geometry);
indexResponse = client.IndexDocument(document);
//End of Scenario #7 -------------------
if (!indexResponse.IsValid) {
throw new Exception($"Error indexinf document: {indexResponse.DebugInformation}");
}
}
catch (Exception ex)
{
Console.WriteLine($"General error: {ex}");
}
}
public class MyDocument {
public MyDocument(int id, Geometry geometry) {
Id = id;
Geometry = geometry;
}
public int Id { get; set; }
public Geometry Geometry { get; set; }
}
Display the source blob
Display the rendered blob
Raw
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Display the source blob
Display the rendered blob
Raw
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment