台電電桿座標 TWD67 -> WGS84 #taipower #crs #osm #import

Makefile

.ONESHELL:

# Dataset: https://data.gov.tw/dataset/33305
DATASET_URL = http://data.taipower.com.tw/opendata/apply/file/d077006/台灣電力公司_電桿桿號及座標資料.zip

# Ref: https://gist.github.com/mutolisp/20b8d11888775932ab7188769e9e8f9d
PROJ_PARAMS_TWD67 = "+proj=tmerc +lat_0=0 +lon_0=121 +k=0.9999 +x_0=250000 +y_0=0 +ellps=aust_SA  +towgs84=-750.739,-359.515,-180.510,0.00003863,0.00001721,0.00000197,0.99998180 +units=m +no_defs" 

# The raw dataset from Taipower company
dataset.zip:
	wget $(DATASET_URL) -O $@

# List of all poles in dataset
raw.csv: dataset.zip
	rm *csv
	unzip $< && rm schema-* manifest.csv
	ls *csv | while read csv; do 
		if [[ $$(head -1 $$csv) =~ TWD67_X ]]; then
			[[ ! -s $@ ]] && head -1 $$csv >>$@
			sed 1d $$csv >>$@
		fi
		rm $$csv
	done

# List of poles with WGS84 lon/lat coordinates
processed.csv: raw.csv
	ogr2ogr $@ $< \
		-lco GEOMETRY=AS_XY \
		-oo X_POSSIBLE_NAMES=TWD67_X \
		-oo Y_POSSIBLE_NAMES=TWD67_Y \
		-s_srs $(PROJ_PARAMS_TWD67) \
		-t_srs "EPSG:4326" \
		-dialect sqlite -sql "select * from input where geometry is not null"

# List of OSM entities ready for upload
osm.list: processed.csv
	sed 1d $< | LANG=C.UTF-8 awk -F',' '
		BEGIN { 
			format_node = "<node id=\x22-%s\x22 version=\x221\x22 lat=\x22%s\x22 lon=\x22%s\x22>"
			format_tag = "<tag k=\x22%s\x22 v=\x22%s\x22/>" 
		}
		{ 
			printf format_node, FNR, $$2, $$1
			printf format_tag, "source", "https://data.gov.tw/dataset/33305"
			printf format_tag, "power", "pole"
			if ($$6!="" && $$8!="") printf format_tag, "ref", $$6 $$8 
			else if ($$6!="")       printf format_tag, "ref", $$6
			print "</node>" 
		}
	' >$@

# Get script used to commit changeset
commit.sh:
	curl -s https://raw.githubusercontent.com/typebrook/helper/dev/tools/osm/osm.api.changeset.commit >$@
	chmod +x $@

# Upload changesets on OSM server (Test server by default)
commit: osm.list commit.sh
	CHECKLIST=uploaded.list
	TOTAL=$$(wc -l <$<)
	# Upload 1000 elements per changeset
	for i in $$(seq 1 1000 $$TOTAL); do 
		# If uploaded, skip this bunch of elements
		grep "^$$i$$" $$CHECKLIST >/dev/null && continue
		# Make a new changeset in background
		{
			COMMIT_FILE=commit.$$i
			echo Processing $$i to $$(($$i+999));
			sed -n "
				1 i <osmChange version=\"0.6\" generator=\"bash script\">
				1 i <create>
				$$i,+999 p
				$$ a </create>
				$$ a </osmChange>
			" $< >$$COMMIT_FILE;
			echo "Import Open data of utility poles from data.gov.tw, licensed by OGDL-v1.
				  This changeset is created by script on https://gist.github.com/1dc1188964a2d559f27ba3899119540d.
			" | SOURCE=https://data.gov.tw/dataset/33305 ./commit.sh $(EXTRA) $$COMMIT_FILE && \
			echo $$i >>$$CHECKLIST && echo Finished || echo Failed;
			rm $$COMMIT_FILE;
		} &
		# Only run limited processes at the same time
		while [ $$(jobs -p | wc -l) -ge 3 ]; do
			sleep 1
		done
	done

# For display on Web Map
taipower-import.pmtiles: processed.csv
	ogr2ogr tmp.gpkg $< \
		-oo X_POSSIBLE_NAMES=X \
		-oo Y_POSSIBLE_NAMES=Y
	ogr2ogr preview.mbtiles tmp.gpkg \
		-dsco MINZOOM=13 \
		-dsco MAXZOOM=13 
	rm tmp.gpkg
	pmtiles-convert tmp.mbtiles taipower-import.pmtiles
	rm tmp.mbtiles

# For other contributors
taipower-import.osc: osm.list
	sed '
		1 i <osmChange version=\"0.6\" generator=\"bash script\">
		1 i <create>
		$$ a </create>
		$$ a </osmChange>
	' $< >$@