Skip to content

Instantly share code, notes, and snippets.

@Zverik
Last active November 4, 2017 12:25
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save Zverik/ddcfaf2da25a3dfda00a3d93a62f218d to your computer and use it in GitHub Desktop.
Save Zverik/ddcfaf2da25a3dfda00a3d93a62f218d to your computer and use it in GitHub Desktop.
Fuel stations import in the UK
2 "tags_changed.brand": "BP -> Shell",
3 "tags_changed.brand": "Esso -> Shell",
3 "tags_changed.brand": "Jet -> Shell",
1 "tags_changed.brand": "Murco -> Shell",
1 "tags_changed.brand": "Red House Garage -> Shell",
1 "tags_changed.brand": "Texaco -> Shell",
16 "tags_changed.brand": "Total -> Shell",
1 "tags_changed.brand": "Tove Service Station -> Shell",
9 "tags_changed.brand": "shell -> Shell",
1 "tags_changed.opening_hours": "05:00-22:00 -> Mo-Su 05:00-22:00",
2 "tags_changed.opening_hours": "06:00-22:00 -> Mo-Su 06:00-22:00",
1 "tags_changed.opening_hours": "06:30-23:00 -> Mo-Su 06:30-22:00",
1 "tags_changed.opening_hours": "24 -> 24/7",
1 "tags_changed.opening_hours": "24/7 -> Mo-Sa 06:00-22:00; Su 07:00-21:00",
1 "tags_changed.opening_hours": "24/7 -> Mo-Su 06:00-23:00",
1 "tags_changed.opening_hours": "24/7 -> Mo-Su 07:00-22:00",
1 "tags_changed.opening_hours": "Mo-Fr 10:00-18:00 -> 24/7",
1 "tags_changed.opening_hours": "Mo-Sa 06:30-22:00;Su 08:00-19:00 -> Mo-Sa 06:30-22:00; Su 07:00-20:00",
1 "tags_changed.opening_hours": "Mo-Su 07:00-23:00 -> Mo-Sa 07:00-23:00; Su 08:00-21:00",
1 "tags_changed.phone": "+44 (0)1332 794127 -> +44 78 8007 8248",
1 "tags_changed.phone": "+44 1766 512934 -> +44 1766 510920",
1 "tags_changed.phone": "+44 2380 552612 -> +44 23 8055 2612",
1 "tags_changed.phone": "+44 2392 413810 -> +44 23 9241 3810",
1 "tags_changed.phone": "+44 2392 693995 -> +44 23 9269 3995",
1 "tags_changed.phone": "+441329845431 -> +44 1329 845431",
1 "tags_changed.phone": "+441428724169 -> +44 1428 724169",
1 "tags_changed.phone": "+442086070900 -> +44 20 8744 0174",
1 "tags_changed.phone": "+442088940700 -> +44 20 8894 0700",
1 "tags_changed.phone": "+442392372014 -> +44 23 9237 2014",
1 "tags_changed.phone": "01606 305890 -> +44 1606 882194",
1 "tags_changed.phone": "01606 590561 -> +44 1606 590561",
1 "tags_changed.phone": "02083482719 -> +44 20 8348 2719",
<!DOCTYPE html>
<html>
<head>
<meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no' />
<style>
body { margin:0; padding:0; }
#map { position:absolute; top:0; bottom:0; width:100%; }
.marker-properties {
border-collapse:collapse;
/*border:1px solid #eee;*/
font-size:11px;
margin:0;
}
.marker-properties th {
white-space:nowrap;
border:1px solid #eee;
padding:5px 10px;
}
.marker-properties td {
border:1px solid #eee;
padding:5px 10px;
}
.marker-properties tr:last-child td,
.marker-properties tr:last-child th {
border-bottom:none;
}
.marker-properties tr:nth-child(even) th,
.marker-properties tr:nth-child(even) td {
background-color:#f7f7f7;
}
.tag-type-heading td {
padding-top: 20px;
padding-left: 2px;
border-style: none;
font-weight:bold;
font-size:1.1em;
}
</style>
<script src='//api.tiles.mapbox.com/mapbox.js/v2.2.2/mapbox.js'></script>
<script src='//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js' ></script>
<script src='//unpkg.com/leaflet-hash@0.2.1/leaflet-hash.js'></script>
<link href='//api.tiles.mapbox.com/mapbox.js/v2.2.2/mapbox.css' rel='stylesheet' />
</head>
<body>
<div id='map'></div>
<script type='text/javascript'>
L.mapbox.accessToken = 'pk.eyJ1IjoibWFwYm94IiwiYSI6ImNpejY4NXFhYTA2bTMyeW44ZG0ybXBkMHkifQ.gUGbDOPUN1v1fTs5SeOR4A';
var map = L.mapbox.map('map');
var layers = {
'Satellite': L.mapbox.tileLayer('mapbox.satellite'),
'OpenStreetMap': L.tileLayer('https://a.tile.openstreetmap.org/{z}/{x}/{y}.png', {
attribution: '&copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors'
})
};
layers['OpenStreetMap'].addTo(map);
L.control.layers(layers, null, {collapsed: false}).addTo(map);
$.getJSON('map.geojson', function(geojson) {
var geojsonLayer = L.mapbox.featureLayer(geojson).addTo(map);
var bounds = geojsonLayer.getBounds();
if (bounds.isValid()) {
map.fitBounds(geojsonLayer.getBounds());
} else {
map.setView([0, 0], 2);
}
var hash = new L.Hash(map);
geojsonLayer.eachLayer(function(l) {
showProperties(l);
});
});
function showProperties(l) {
var properties = l.toGeoJSON().properties;
var popupContent = $("<div></div>");
osm_object_link = 'new node';
if (properties['action'] == 'modify') {
osm_id = properties['osm_type'] + '/' + properties['osm_id'];
osm_object_link = '<a href="http://www.openstreetmap.org/'+osm_id+'">' +
osm_id + '</a></p>'
}
popupContent.html(
'<p><span style="color:' + properties['marker-color'] + ';">' +
properties['action'] +
'</span> ' + osm_object_link + '</p>'
);
if (properties['ref_coords'])
popupContent.append('<p>Dataset position:' + properties['ref_coords'] + '</p>');
if (properties['ref_distance'])
popupContent.append('<p>Distance from OSM to dataset:' + properties['ref_distance'] + ' m</p>');
var table = document.createElement('table');
table.setAttribute('class', 'marker-properties display')
lastTagType = '';
for (var key in properties) {
if (key.startsWith('tag')) {
parts = key.split('.');
tag_type = parts[0];
osm_key = parts[1];
if (tag_type!=lastTagType) {
var tr = createTableHeading(tag_type);
table.appendChild(tr);
lastTagType = tag_type;
}
var tr = createTableRows(osm_key, properties[key]);
table.appendChild(tr);
}
}
if (table) popupContent.append(table);
l.bindPopup(popupContent[0]);
}
function createTableHeading(tag_type) {
if (tag_type == 'tags') {
headingText = 'Tags:';
} else if (tag_type == 'tags_changed') {
headingText = 'Tags to change:';
} else if (tag_type == 'tags_removed') {
headingText = 'Tags to remove:';
} else if (tag_type == 'tags_new') {
headingText = 'New tags to add:';
} else {
headingText = tag_type; //unknown type
}
var tr = document.createElement('tr');
var att = document.createAttribute('class');
att.value = 'tag-type-heading';
tr.setAttributeNode(att);
var td = document.createElement('td');
heading = document.createTextNode(headingText);
td.appendChild(heading);
var att = document.createAttribute("colspan");
att.value = "2";
td.setAttributeNode(att);
tr.appendChild(td);
return tr;
}
function createTableRows(key, value) {
var tr = document.createElement('tr');
var th = document.createElement('th');
var td = document.createElement('td');
key = document.createTextNode(key);
value = document.createTextNode(value);
th.appendChild(key);
td.appendChild(value);
tr.appendChild(th);
tr.appendChild(td);
return tr
}
</script>
</body>
</html>
Display the source blob
Display the rendered blob
Raw
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
import json
import codecs
import re
from collections import defaultdict
source = 'Navads'
dataset_id = 'navads_shell'
query = [('amenity', 'fuel')]
master_tags = ('brand', 'phone', 'opening_hours')
max_distance = 50
max_request_boxes = 3
def dataset(fileobj):
def format_phone(ph):
if ph and len(ph) == 13 and ph[:3] == '+44':
if (ph[3] == '1' and ph[4] != '1' and ph[5] != '1') or ph[3:7] == '7624':
return ' '.join([ph[:3], ph[3:7], ph[7:]])
elif ph[3] in ('1', '3', '8', '9'):
return ' '.join([ph[:3], ph[3:6], ph[6:9], ph[9:]])
else:
return ' '.join([ph[:3], ph[3:5], ph[5:9], ph[9:]])
return ph
def make_wd_ranges(r):
wd = ['Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa', 'Su']
res = wd[r[0]]
in_range = False
for i in range(1, len(r)+1):
if i < len(r) and r[i] == r[i-1] + 1:
in_range = True
else:
if in_range:
res += '-' + wd[r[i-1]]
in_range = False
if i < len(r):
res += ',' + wd[r[i]]
return res
def parse_hours(h):
if not h:
return None
WD = {x: i for i, x in enumerate([
'MONDAY', 'TUESDAY', 'WEDNESDAY', 'THURSDAY', 'FRIDAY', 'SATURDAY', 'SUNDAY'
])}
days = defaultdict(list)
for d in h.split(';'):
parts = re.findall(r'([A-Z]+)=([0-9:-]+)', d)
if len(set([p[0] for p in parts])) != 1:
raise Exception('Parts format fail: {}'.format(d))
days[','.join([p[1] for p in parts])].append(WD[parts[0][0]])
res = []
for time, wd in sorted(days.items(), key=lambda x: min(x[1])):
res.append(' '.join([make_wd_ranges(wd), time]))
if res[0] == 'Mo-Su 00:00-23:59':
return '24/7'
return '; '.join(res).replace('23:59', '24:00')
source = json.load(codecs.getreader('utf-8-sig')(fileobj))
data = []
for el in source['Locations']:
if not el['location']:
continue
coords = [float(x) for x in el['location'].split(',')]
tags = {
'amenity': 'fuel',
'brand': el['name'],
'addr:postcode': el['address_zip'] or None,
'phone': format_phone('+'+str(el['phone'])),
'opening_hours': parse_hours(el['daily_hours']),
}
if (el['address_street'] and el['address_number'] and
not re.search(r'^([ABCDM]\d+|Junction)', el['address_street']) and
'Ln' not in el['address_street'] and 'A' not in el['address_number']):
tags['addr:street'] = el['address_street']
tags['addr:housenumber'] = el['address_number']
data.append(SourcePoint(el['place_id'], coords[0], coords[1], tags))
return data
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment