Skip to content

Instantly share code, notes, and snippets.

@nickpeihl

nickpeihl/Console

Created March 11, 2021 01:06
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save nickpeihl/343fa67d43e22f4afc3a6fd88c2ee0f6 to your computer and use it in GitHub Desktop.
Save nickpeihl/343fa67d43e22f4afc3a6fd88c2ee0f6 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
README.md

Import OSM into Elasticsearch

osmium cat -o andorra-latest.opl andorra-latest.osm.pbf
cat andorra-latest.opl | logstash -f opl-filter.conf

See ./Console for mappings, ingest processing, and scripts (copied directly from my working Kibana console)

Does not yet create geo_shapes from relations.

Raw
Console
GET _search
{
"query": {
"match_all": {}
}
}
GET openstreetmap-2021.03.10/_mapping
GET openstreetmap-ways/_mapping
DELETE _license
GET _license
GET osm-monaco/_search
DELETE _ingest/pipeline/osm_way_node_coord
GET openstreetmap-ways/_doc/w104330539
POST _ingest/pipeline/_simulate?verbose=true
{
"pipeline": {
"description": "is closed",
"processors": [
{
"script": {
"source": """
def coordsize = ctx._source.coordinates;
ctx._source.coordsize = coordsize;
"""
}
}
]
},
"docs": [
{
"_index": "openstreetmap-ways",
"_type": "_doc",
"_id": "w104330539",
"_version": 1,
"_score": null,
"_source": {
"changeset_id": "0",
"coordinates": [
"42.608208,1.7153049",
"42.608143,1.7152941",
"42.608078,1.7152251",
"42.608044,1.7152901",
"42.607975,1.7153682",
"42.607964,1.7153776",
"42.607983,1.7154207",
"42.608013,1.7154796",
"42.60811,1.7155403",
"42.608173,1.7156391",
"42.608154,1.715783",
"42.60817,1.716625",
"42.608284,1.7170587",
"42.60851,1.7175467",
"42.608658,1.7176645",
"42.60877,1.7175298",
"42.609016,1.7176476",
"42.609127,1.7182198",
"42.609215,1.719078",
"42.60921,1.7191267",
"42.609165,1.720542",
"42.609165,1.7222861",
"42.609165,1.7225109",
"42.60925,1.7226287",
"42.609608,1.722006",
"42.609894,1.7224604",
"42.610302,1.7228138",
"42.6105,1.722595",
"42.610367,1.7219723",
"42.61035,1.7213835",
"42.6106,1.721518",
"42.610687,1.7220229",
"42.610847,1.7223594",
"42.611034,1.722006",
"42.610886,1.7214676",
"42.611084,1.7215517",
"42.610897,1.720744",
"42.611156,1.7205757",
"42.61122,1.7212824",
"42.611988,1.7214001",
"42.61201,1.721201",
"42.612076,1.7206599",
"42.612125,1.7199026",
"42.612,1.7192631",
"42.61196,1.7188761",
"42.61215,1.7185563",
"42.612274,1.7184217",
"42.612297,1.7183172",
"42.612347,1.7181188",
"42.612186,1.7176476",
"42.612026,1.7167894",
"42.61189,1.7162678",
"42.61195,1.7156619",
"42.61175,1.7155273",
"42.611652,1.7152917",
"42.61138,1.7150561",
"42.611095,1.7150561",
"42.610798,1.7153085",
"42.61054,1.7151738",
"42.61045,1.7147869",
"42.609955,1.7148879",
"42.609745,1.7151066",
"42.609436,1.7149384",
"42.60898,1.7149888",
"42.60872,1.7152749",
"42.608482,1.7154204",
"42.60839,1.7153081",
"42.608208,1.7153049"
],
"message": "w104330539 v8 dV c0 t2021-02-17T20:15:19Z i0 u Tname=Estany%20%Primer%20%de%20%Juclar,natural=water,water=lake,wikidata=Q21330259 Nn1203763744,n3870464778,n3870464775,n3870464774,n3870464771,n3870464770,n3870464772,n3870464773,n3870464776,n1203763480,n1203763176,n526432183,n1203762562,n1203763008,n1203762640,n1203764267,n1203763927,n1203763693,n1203763426,n2648192365,n1203763137,n2648192257,n1203762855,n1203763399,n1203763118,n1203762825,n1203764389,n1203764107,n1203763794,n1203763555,n1203763263,n1203763944,n1203763716,n1203763456,n1203763155,n1203762883,n1203764417,n1203764150,n1203763816,n1203764404,n2648394344,n1203764134,n1203763804,n1203763573,n1203763282,n1203763023,n1203762663,n2648394411,n1203764056,n1203763033,n1203762679,n1203764290,n1203763957,n1203763727,n1203763467,n1203764185,n1203763847,n1203763610,n1203763335,n1203763051,n1203762716,n1203764318,n1203764001,n1203762705,n1203764305,n1203763980,n5043826018,n1203763744",
"version": "8",
"userid": "0",
"@timestamp": "2021-02-17T20:15:19Z",
"deleted": "V",
"nodes": [
"n1203763744",
"n3870464778",
"n3870464775",
"n3870464774",
"n3870464771",
"n3870464770",
"n3870464772",
"n3870464773",
"n3870464776",
"n1203763480",
"n1203763176",
"n526432183",
"n1203762562",
"n1203763008",
"n1203762640",
"n1203764267",
"n1203763927",
"n1203763693",
"n1203763426",
"n2648192365",
"n1203763137",
"n2648192257",
"n1203762855",
"n1203763399",
"n1203763118",
"n1203762825",
"n1203764389",
"n1203764107",
"n1203763794",
"n1203763555",
"n1203763263",
"n1203763944",
"n1203763716",
"n1203763456",
"n1203763155",
"n1203762883",
"n1203764417",
"n1203764150",
"n1203763816",
"n1203764404",
"n2648394344",
"n1203764134",
"n1203763804",
"n1203763573",
"n1203763282",
"n1203763023",
"n1203762663",
"n2648394411",
"n1203764056",
"n1203763033",
"n1203762679",
"n1203764290",
"n1203763957",
"n1203763727",
"n1203763467",
"n1203764185",
"n1203763847",
"n1203763610",
"n1203763335",
"n1203763051",
"n1203762716",
"n1203764318",
"n1203764001",
"n1203762705",
"n1203764305",
"n1203763980",
"n5043826018",
"n1203763744"
],
"tagstore": {
"natural": "water",
"name": "Estany%20%Primer%20%de%20%Juclar",
"water": "lake",
"wikidata": "Q21330259"
},
"osmid": "w104330539",
"@version": "1",
"host": "Nicks-MacBook-Pro-2.local",
"tag": {
"natural": "water"
}
},
"fields": {
"@timestamp": [
"2021-02-17T20:15:19.000Z"
],
"coordinates": [
"42.608208,1.7153049",
"42.608143,1.7152941",
"42.608078,1.7152251",
"42.608044,1.7152901",
"42.607975,1.7153682",
"42.607964,1.7153776",
"42.607983,1.7154207",
"42.608013,1.7154796",
"42.60811,1.7155403",
"42.608173,1.7156391",
"42.608154,1.715783",
"42.60817,1.716625",
"42.608284,1.7170587",
"42.60851,1.7175467",
"42.608658,1.7176645",
"42.60877,1.7175298",
"42.609016,1.7176476",
"42.609127,1.7182198",
"42.609215,1.719078",
"42.60921,1.7191267",
"42.609165,1.720542",
"42.609165,1.7222861",
"42.609165,1.7225109",
"42.60925,1.7226287",
"42.609608,1.722006",
"42.609894,1.7224604",
"42.610302,1.7228138",
"42.6105,1.722595",
"42.610367,1.7219723",
"42.61035,1.7213835",
"42.6106,1.721518",
"42.610687,1.7220229",
"42.610847,1.7223594",
"42.611034,1.722006",
"42.610886,1.7214676",
"42.611084,1.7215517",
"42.610897,1.720744",
"42.611156,1.7205757",
"42.61122,1.7212824",
"42.611988,1.7214001",
"42.61201,1.721201",
"42.612076,1.7206599",
"42.612125,1.7199026",
"42.612,1.7192631",
"42.61196,1.7188761",
"42.61215,1.7185563",
"42.612274,1.7184217",
"42.612297,1.7183172",
"42.612347,1.7181188",
"42.612186,1.7176476",
"42.612026,1.7167894",
"42.61189,1.7162678",
"42.61195,1.7156619",
"42.61175,1.7155273",
"42.611652,1.7152917",
"42.61138,1.7150561",
"42.611095,1.7150561",
"42.610798,1.7153085",
"42.61054,1.7151738",
"42.61045,1.7147869",
"42.609955,1.7148879",
"42.609745,1.7151066",
"42.609436,1.7149384",
"42.60898,1.7149888",
"42.60872,1.7152749",
"42.608482,1.7154204",
"42.60839,1.7153081",
"42.608208,1.7153049"
]
},
"sort": [
1613592919000
]
}
]
}
POST _ingest/pipeline/osm_way_node_coord2/_simulate?verbose=true
{
"docs": [
{
"_source": {
"osmid": "n1203763744"
}
}
]
}
PUT _ingest/pipeline/osm_way_node_coord
{
"processors": [
{
"enrich": {
"policy_name": "osm_way_nodes",
"field": "_ingest._value",
"target_field": "_ingest.enrich"
}
},
{
"append": {
"field": "_source.coordinates",
"value": "[{{_ingest.enrich.location.lon}},{{_ingest.enrich.location.lat}}]",
"allow_duplicates": true
}
},
{
"join": {
"field": "_source.coordinates",
"separator": ","
}
},
{
"set": {
"field": "_source.linestringjson",
"value": "{\"type\": \"linestring\", \"coordinates\": [{{_source.coordinates}}] }",
"ignore_empty_value": true
}
},
{
"json": {
"field": "_source.linestringjson",
"target_field": "_source.geometry"
}
}
]
}
PUT _ingest/pipeline/osm_way_nodes
{
"processors": [
{
"foreach": {
"field": "nodes",
"ignore_missing": true,
"processor": {
"pipeline": {
"name": "osm_way_node_coord"
}
}
}
},
{
"remove": {
"field": ["coordinates", "linestringjson"],
"ignore_missing": true
}
}
]
}
DELETE _enrich/policy/osm_way_nodes
PUT _enrich/policy/osm_way_nodes
{
"match": {
"indices": "openstreetmap-2021.03.10",
"match_field": "osmid",
"enrich_fields": ["location"]
}
}
GET openstreetmap-2021-03-10/_mapping
POST _enrich/policy/osm_way_nodes/_execute
GET _enrich/policy/osm_way_nodes
GET _tasks/c5FLo7RPRiyO3WfFpjA4XQ:9703852
DELETE openstreetmap-ways
POST openstreetmap-2021.03.10/_search
{
"query": {
"term": {
"osmid": {
"value": "w104330539"
}
}
}
}
GET _tasks/c5FLo7RPRiyO3WfFpjA4XQ:14706964
DELETE openstreetmap-ways
GET openstreetmap-ways/_mapping
POST _reindex?wait_for_completion=false
{
"source": {
"index": "openstreetmap-2021.03.10",
"query": {
"term": {
"type": {
"value": "w"
}
}
}
},
"dest": {
"index": "openstreetmap-ways",
"pipeline": "osm_way_nodes"
}
}
PUT _ingest/pipeline/osm_nodes
{
"processors": [
{
"grok": {
"field": "_ingest._value",
"patterns": [
"%{OSMID:_ingest.node.nodeid}(x%{NUMBER:_ingest.node.longitude:float})?(y%{NUMBER:_ingest.node.latitude:float})?"
],
"pattern_definitions": {
"OSMID": "[n|r|w][0-9]+",
"NODES": "(%{OSMID},?)+",
"ROLE": "%{NOTSPACE}+",
"MEMBERS": "(%{OSMID}@%{ROLE}?,?)+"
}
}
}
]
}
PUT _ingest/pipeline/osm_members
{
"processors": [
{
"grok": {
"field": "_ingest._value",
"patterns": [
"%{OSMID:_ingest.member.memberid}@(%{ROLE:_ingest.member.role})?"
],
"pattern_definitions": {
"OSMID": "[n|r|w][0-9]+",
"NODES": "(%{OSMID},?)+",
"ROLE": "%{NOTSPACE}+",
"MEMBERS": "(%{OSMID}@%{ROLE}?,?)+"
}
}
},
{
"set": {
"field": "_ingest._value",
"value": {
"id": "{{_ingest.member.memberid}}"
}
}
},
{
"set": {
"field": "_ingest._value.role",
"value": "{{_ingest.member.role}}",
"ignore_empty_value": true
}
},
{
"remove": {
"field": "_ingest.member"
}
}
]
}
PUT _ingest/pipeline/osm_tags
{
"processors": [
{
"kv": {
"field": "taglist",
"field_split": ",",
"value_split": "=",
"ignore_missing": true,
"include_keys": [
"addr:housenumber",
"admin_level",
"aerialway",
"aeroway",
"amenity",
"area:aeroway",
"barrier",
"boundary",
"building",
"building:part",
"claimed_by",
"highway",
"historic",
"landuse",
"leisure",
"man_made",
"natural",
"place",
"public_transport",
"railway",
"ref",
"route",
"shop",
"sport",
"tourism",
"type",
"waterway",
"wetland"
],
"target_field": "tag"
}
},
{
"kv": {
"field": "taglist",
"field_split": ",",
"value_split": "=",
"ignore_missing": true,
"target_field": "tagstore"
}
},
{
"remove": {
"field": "taglist",
"ignore_missing": true
}
}
]
}
PUT _ingest/pipeline/osm
{
"description": "OpenStreetMap OPL pipeline",
"processors": [
{
"grok": {
"field": "message",
"patterns": [
"%{OSMID:_id}%{SPACE}v%{NUMBER:version}%{SPACE}d(?<deleted>[D|V])%{SPACE}c%{INT:changeset_id}(%{SPACE}t%{TIMESTAMP_ISO8601:@timestamp})?%{SPACE}i%{USERNAME:userid}%{SPACE}u(%{USERNAME:username})?%{SPACE}T(?<taglist>[%{NOTSPACE}+=%{NOTSPACE}+]+)?(%{SPACE}x%{NUMBER:longitude:float})?(%{SPACE}y%{NUMBER:latitude:float})?(%{SPACE}N%{NODES:nodes})?(%{SPACE}M%{MEMBERS:members})?"
],
"pattern_definitions": {
"OSMTYPE": "[n|r|w]",
"OSMID": "%{OSMTYPE}[0-9]+",
"NODES": "(%{OSMID}(x%{NUMBER})?(y%{NUMBER})?,?)+",
"ROLE": "%{NOTSPACE}+",
"MEMBERS": "(%{OSMID}@%{ROLE}?,?)+"
}
}
},
{
"grok": {
"field": "_id",
"patterns": ["%{OSMTYPE:type}"],
"pattern_definitions": {
"OSMTYPE": "[n|r|w]",
"OSMID": "%{OSMTYPE}[0-9]+",
"NODES": "(%{OSMID}(x%{NUMBER})?(y%{NUMBER})?,?)+",
"ROLE": "%{NOTSPACE}+",
"MEMBERS": "(%{OSMID}@%{ROLE}?,?)+"
}
}
},
{
"set": {
"field": "osmid",
"copy_from": "_id"
}
},
{
"rename": {
"field": "longitude",
"target_field": "location.lon",
"ignore_missing": true
}
},
{
"rename": {
"field": "latitude",
"target_field": "location.lat",
"ignore_missing": true
}
},
{
"split": {
"field": "members",
"separator": ",",
"ignore_missing": true
}
},
{
"foreach": {
"field": "members",
"processor": {
"pipeline": {
"name": "osm_members"
}
},
"ignore_missing": true
}
},
{
"split": {
"field": "nodes",
"separator": ",",
"ignore_missing": true
}
},
{
"pipeline": {
"name": "osm_tags"
}
}
]
}
PUT _index_template/openstreetmap
{
"index_patterns": [
"openstreetmap-*"
],
"template": {
"settings": {
"number_of_shards": 1
},
"mappings": {
"_source": {
"enabled": true
},
"properties": {
"@timestamp": {
"type": "date"
},
"@version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"location": {
"type": "geo_point"
},
"changeset_id": {
"type": "integer"
},
"deleted": {
"type": "keyword"
},
"type": {
"type": "keyword"
},
"host": {
"type": "keyword"
},
"members": {
"properties": {
"id": {
"type": "keyword"
},
"role": {
"type": "keyword"
}
}
},
"nodes": {
"type": "keyword"
},
"coordinates": {
"type": "keyword",
"store": true
},
"geometry": {
"type": "geo_shape"
},
"message": {
"type": "text"
},
"osmid": {
"type": "keyword"
},
"tag": {
"properties": {
"natural": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"route": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"tagstore": {
"type": "flattened"
},
"userid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "long"
}
}
}
}
}
POST _ingest/pipeline/_simulate?verbose=true
{
"pipeline": {
"processors": [
{
"grok": {
"field": "message",
"patterns": [
"%{OSMID:_id}%{SPACE}v%{NUMBER:version}%{SPACE}d(?<deleted>[D|V])%{SPACE}c%{INT:changeset_id}(%{SPACE}t%{TIMESTAMP_ISO8601:@timestamp})?%{SPACE}i%{USERNAME:userid}%{SPACE}u(%{USERNAME:username})?%{SPACE}T(?<taglist>[%{NOTSPACE}+=%{NOTSPACE}+]+)?(%{SPACE}x%{NUMBER:longitude:float})?(%{SPACE}y%{NUMBER:latitude:float})?(%{SPACE}N%{NODES:nodes})?(%{SPACE}M%{MEMBERS:members})?"
],
"pattern_definitions": {
"OSMTYPE": "[n|r|w]",
"OSMID": "%{OSMTYPE}[0-9]+",
"NODES": "(%{OSMID}(x%{NUMBER})?(y%{NUMBER})?,?)+",
"ROLE": "%{NOTSPACE}+",
"MEMBERS": "(%{OSMID}@%{ROLE}?,?)+"
}
}
},
{
"grok": {
"field": "_id",
"patterns": ["%{OSMTYPE:type}"],
"pattern_definitions": {
"OSMTYPE": "[n|r|w]",
"OSMID": "%{OSMTYPE}[0-9]+",
"NODES": "(%{OSMID}(x%{NUMBER})?(y%{NUMBER})?,?)+",
"ROLE": "%{NOTSPACE}+",
"MEMBERS": "(%{OSMID}@%{ROLE}?,?)+"
}
}
},
{
"set": {
"field": "osmid",
"copy_from": "_id"
}
},
{
"rename": {
"field": "longitude",
"target_field": "location.lon",
"ignore_missing": true
}
},
{
"rename": {
"field": "latitude",
"target_field": "location.lat",
"ignore_missing": true
}
},
{
"split": {
"field": "members",
"separator": ",",
"ignore_missing": true
}
},
{
"foreach": {
"field": "members",
"processor": {
"pipeline": {
"name": "osm_members"
}
},
"ignore_missing": true
}
},
{
"split": {
"field": "nodes",
"separator": ",",
"ignore_missing": true
}
},
{
"pipeline": {
"name": "osm_tags"
}
}
]
},
"docs": [
{
"_source": {
"message": "w4634293 v16 dV c0 t2018-03-05T19:49:46Z i0 u Tref=WA%20%520,lanes=2,oneway=yes,bicycle=no,highway=motorway,old_ref=PSH%20%1-EP,maxspeed=60%20%mph Nn335444348,n4240314157"
}
}
]
}
Raw
opl-filter.conf
input { stdin {} }
output {
elasticsearch {
index => "openstreetmap-2021.03.10"
pipeline => "osm"
hosts => ["localhost:9400"]
user => "elastic"
password => "<my_secret_password>"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment