Skip to content

Instantly share code, notes, and snippets.

@junichim junichim/README.md
Last active May 13, 2019

Embed
What would you like to do?
オフラインマップ生成用のスクリプト類
#!/bin/bash
#
# create files for offline map.
#
# Junichi MORI, 2019/5/1
BIN_DIR=/home/osm/bin/offlinemap
MAP_SCRIPT=doMapCreator
GH_SCRIPT=doGh
UPLOAD_SCRIPT=uploadS3
WORK_DIR=/home/osm/work/offlinemap
MAP_DIR="$WORK_DIR/maps/v4/asia/japan"
MAP_FILE=kansai.map
MAP_OUT=ise.map
GHZ_DIR=$WORK_DIR/data/asia/japan/kansai/kansai-gh
GHZ_FILE=kansai.ghz
GHZ_OUT=ise.ghz
TS_FILE=map_timestamp
RESULT_DIR=$WORK_DIR/result
if [ ! -f $BIN_DIR/$MAP_SCRIPT ]; then
echo "no "$MAP_SCRIPT" file"
exit 1
fi
if [ ! -f $BIN_DIR/$GH_SCRIPT ]; then
echo "no "$GH_SCRIPT" file"
exit 1
fi
# 1. craete map file
echo create map file
echo $BIN_DIR/$MAP_SCRIPT
$BIN_DIR/$MAP_SCRIPT
if [ $? != 0 ]; then
echo "error in map-creation"
exit 1
fi
# 2. craete ghz file
echo create ghz file
echo $BIN_DIR/$GH_SCRIPT
$BIN_DIR/$GH_SCRIPT
if [ $? != 0 ]; then
echo "error in graphhopper"
exit 1
fi
# 3. copy created file
echo copy created files
TS_OLD=`cat $RESULT_DIR/$TS_FILE`
# file copy
if [ -f $MAP_DIR/$MAP_FILE ]; then
cp -p $MAP_DIR/$MAP_FILE $RESULT_DIR/$MAP_OUT
# timestamp file
date --reference=$RESULT_DIR/$MAP_OUT --iso-8601=seconds > $RESULT_DIR/$TS_FILE
fi
if [ -f $GHZ_DIR/$GHZ_FILE ]; then
cp -p $GHZ_DIR/$GHZ_FILE $RESULT_DIR/$GHZ_OUT
fi
# 4. upload to S3 bucket
if [ $TS_OLD == `cat $RESULT_DIR/$TS_FILE` ]; then
echo "map file timestamp is not updated. so skip S3 upload."
else
echo upload S3 bucket
echo $BIN_DIR/$UPLOAD_SCRIPT
$BIN_DIR/$UPLOAD_SCRIPT
if [ $? != 0 ]; then
echo "error to upload S3"
exit 1
fi
fi
exit 0
#!/bin/bash
#
# do graphhopper to create ghz file
#
# written by Junichi MORI
# 2019/4/23
#
GH_BIN=$HOME/bin/graphhopper
export MAVEN_HOME=$GH_BIN/maven
WORK_DIR="/home/osm/work/offlinemap"
PBF_DIR="$WORK_DIR/data/asia/japan/kansai"
PBF_FILE="$PBF_DIR/kansai.pbf"
GH_DIR="$PBF_DIR/`basename $PBF_DIR`""-gh"
GHZ_FILE="`basename $PBF_DIR`"".ghz"
echo target output dir : $GH_DIR
echo target output file: $GHZ_FILE
# 1. preparation
if [ -d $GH_DIR ]; then
rm -rf $GH_DIR
fi
# 2. create
cd $GH_BIN
# for 0.12.0
echo $GH_BIN/graphhopper.sh --action import --input $PBF_FILE --config $WORK_DIR/gh_config.yml
./graphhopper.sh --action import --input $PBF_FILE --config $WORK_DIR/gh_config.yml
if [ $? != 0 ]; then
echo "graphhopper import failed."
exit 1
fi
# 3. zip
cd $GH_DIR
pwd
echo zip $GHZ_FILE ./*
zip $GHZ_FILE ./*
exit 0
#!/bin/bash
#
# do mapsforge/map-creator
#
# written by Junichi MORI
# 2019/4/23
#
BIN_DIR="/home/osm/bin/mapsforge-creator"
MAP_CREATOR="map-creator"
$BIN_DIR/$MAP_CREATOR asia/japan/kansai ram ja ise.poly
exit $?
#!/bin/bash
# Mapsforge map creation (with coastlines)
# using OpenStreetMap data from Geofabrik
#
# https://github.com/mapsforge/mapsforge/blob/master/docs/MapCreation.md
#
# Written by devemux86
# Configuration
# http://wiki.openstreetmap.org/wiki/Osmosis
#[ $OSMOSIS_HOME ] || OSMOSIS_HOME="$HOME/programs/osmosis"
[ $OSMOSIS_HOME ] || OSMOSIS_HOME="$HOME/bin/osmosis-0.47"
# add by Junichi MORI, 2019/5/1
[ $WORK_HOME ] || WORK_HOME="$HOME/work/offlinemap"
#[ $DATA_PATH ] || DATA_PATH="$HOME/mapsforge/data"
[ $DATA_PATH ] || DATA_PATH="$WORK_HOME/data"
#[ $MAPS_PATH ] || MAPS_PATH="$HOME/mapsforge/maps"
[ $MAPS_PATH ] || MAPS_PATH="$WORK_HOME/maps"
#[ $POIS_PATH ] || POIS_PATH="$HOME/mapsforge/pois"
[ $POIS_PATH ] || POIS_PATH="$WORK_HOME/pois"
[ $PROGRESS_LOGS ] || PROGRESS_LOGS="true"
[ $TAG_VALUES ] || TAG_VALUES="false"
[ $THREADS ] || THREADS="1"
#[ $DAYS ] || DAYS="30"
[ $DAYS ] || DAYS="27"
# added by Junichi MORI, 2019/5/1
SKIP_POI_CREATION="true"
# =========== DO NOT CHANGE AFTER THIS LINE. ===========================
# Below here is regular code, part of the file. This is not designed to
# be modified by users.
# ======================================================================
if [ $# -lt 2 ]; then
#echo "Usage: $0 continent/country[/region] ram|hd [lang,...]"
#echo "Example: $0 europe/germany/berlin ram en,de,fr,es"
echo "Usage: $0 continent/country[/region] ram|hd [lang,...] [poly_file]"
echo "Example: $0 europe/germany/berlin ram en,de,fr,es ise.poly"
exit
fi
cd "$(dirname "$0")"
NAME="$(basename "$1")"
WORK_PATH="$DATA_PATH/$1"
if [ "$TAG_VALUES" = "true" ]; then
MAPS_PATH="$MAPS_PATH/v5"
else
[ $3 ] && MAPS_PATH="$MAPS_PATH/v4" || MAPS_PATH="$MAPS_PATH/v3"
fi
MAPS_PATH="$(dirname "$MAPS_PATH/$1")"
POIS_PATH="$(dirname "$POIS_PATH/$1")"
# check 2nd argument
if [ $2 != "ram" -a $2 != "hd" ]; then
echo "2nd argument is invalid. must be ram or hd. : " $2
exit
fi
# Check map date
if [ -f "$MAPS_PATH/$NAME.map" ]; then
if [ $(find "$MAPS_PATH/$NAME.map" -mtime -$DAYS) ]; then
echo "$MAPS_PATH/$NAME.map exists and is newer than $DAYS days."
exit
fi
fi
# Pre-process
rm -rf "$WORK_PATH"
mkdir -p "$WORK_PATH"
if [ "$SKIP_MAP_CREATION" != "true" ]; then
mkdir -p "$MAPS_PATH"
fi
if [ "$SKIP_POI_CREATION" != "true" ]; then
mkdir -p "$POIS_PATH"
fi
# Download land
if [ -f "$DATA_PATH/land-polygons-split-4326/land_polygons.shp" ] && [ $(find "$DATA_PATH/land-polygons-split-4326/land_polygons.shp" -mtime -$DAYS) ]; then
echo "Land polygons exist and are newer than $DAYS days."
else
echo "Downloading land polygons..."
rm -rf "$DATA_PATH/land-polygons-split-4326"
rm -f "$DATA_PATH/land-polygons-split-4326.zip"
wget -nv -N -P "$DATA_PATH" http://data.openstreetmapdata.com/land-polygons-split-4326.zip || exit 1
unzip -oq "$DATA_PATH/land-polygons-split-4326.zip" -d "$DATA_PATH"
fi
# Download data
echo "Downloading $1..."
wget -nv -N -P "$WORK_PATH" https://download.geofabrik.de/$1-latest.osm.pbf || exit 1
wget -nv -N -P "$WORK_PATH" https://download.geofabrik.de/$1-latest.osm.pbf.md5 || exit 1
(cd "$WORK_PATH" && exec md5sum -c "$NAME-latest.osm.pbf.md5") || exit 1
wget -nv -N -P "$WORK_PATH" https://download.geofabrik.de/$1.poly || exit 1
# modified by Junichi MORI, 2019/5/1
# Bounds
[ $4 ] && POLY_FILE="$WORK_HOME/$4" || POLY_FILE="$WORK_PATH/$NAME.poly"
echo "Poly file is " $POLY_FILE
#BBOX=$(perl poly2bb.pl "$WORK_PATH/$NAME.poly")
BBOX=$(perl poly2bb.pl "$POLY_FILE")
BBOX=(${BBOX//,/ })
BOTTOM=${BBOX[0]}
LEFT=${BBOX[1]}
TOP=${BBOX[2]}
RIGHT=${BBOX[3]}
# Start position
# modified by Junichi MORI, 2019/5/1
#CENTER=$(perl poly2center.pl "$WORK_PATH/$NAME.poly")
CENTER=$(perl poly2center.pl "$POLY_FILE")
CENTER=(${CENTER//,/ })
LAT=${CENTER[0]}
LON=${CENTER[1]}
# added by Junichi MORI, 2019/5/1
# extract only bounding box
BOUNDED_PBF="$NAME.pbf"
echo $BOUNDED_PBF
CMD="$OSMOSIS_HOME/bin/osmosis --rb file=$WORK_PATH/$NAME-latest.osm.pbf \
--bb left=$LEFT right=$RIGHT top=$TOP bottom=$BOTTOM completeWays=yes \
--wb file=$WORK_PATH/$BOUNDED_PBF"
echo $CMD
$CMD
# Land
ogr2ogr -overwrite -progress -skipfailures -clipsrc $LEFT $BOTTOM $RIGHT $TOP "$WORK_PATH/land.shp" "$DATA_PATH/land-polygons-split-4326/land_polygons.shp"
python shape2osm.py -l "$WORK_PATH/land" "$WORK_PATH/land.shp"
# Sea
cp sea.osm "$WORK_PATH"
sed -i "s/\$BOTTOM/$BOTTOM/g" "$WORK_PATH/sea.osm"
sed -i "s/\$LEFT/$LEFT/g" "$WORK_PATH/sea.osm"
sed -i "s/\$TOP/$TOP/g" "$WORK_PATH/sea.osm"
sed -i "s/\$RIGHT/$RIGHT/g" "$WORK_PATH/sea.osm"
# Merge
# modified by Junichi MORI, 2019/5/1
#CMD="$OSMOSIS_HOME/bin/osmosis --rb file=$WORK_PATH/$NAME-latest.osm.pbf \
# --rx file=$WORK_PATH/sea.osm --s --m"
CMD="$OSMOSIS_HOME/bin/osmosis --rb file=$WORK_PATH/$BOUNDED_PBF \
--rx file=$WORK_PATH/sea.osm --s --m"
for f in $WORK_PATH/land*.osm; do
CMD="$CMD --rx file=$f --s --m"
done
CMD="$CMD --wb file=$WORK_PATH/merge.pbf omitmetadata=true"
echo $CMD
$CMD
# Map
if [ "$SKIP_MAP_CREATION" != "true" ]; then
CMD="$OSMOSIS_HOME/bin/osmosis --rb file=$WORK_PATH/merge.pbf"
[ $MAP_TRANSFORM_FILE ] && CMD="$CMD --tt file=$MAP_TRANSFORM_FILE"
CMD="$CMD --mw file=$WORK_PATH/$NAME.map \
type=$2 \
bbox=$BOTTOM,$LEFT,$TOP,$RIGHT \
map-start-position=$LAT,$LON \
map-start-zoom=8 \
tag-values=$TAG_VALUES \
threads=$THREADS \
progress-logs=$PROGRESS_LOGS"
[ $3 ] && CMD="$CMD preferred-languages=$3"
[ $MAP_TAG_CONF_FILE ] && CMD="$CMD tag-conf-file=$MAP_TAG_CONF_FILE"
echo $CMD
$CMD || exit 1
# Check map size
if [ -f "$MAPS_PATH/$NAME.map" ]; then
OLD_SIZE=$(wc -c < "$MAPS_PATH/$NAME.map")
NEW_SIZE=$(wc -c < "$WORK_PATH/$NAME.map")
if [ $NEW_SIZE -lt $(($OLD_SIZE * 70 / 100)) ]; then
echo "$WORK_PATH/$NAME.map creation is significantly smaller."
exit 1
fi
fi
mv "$WORK_PATH/$NAME.map" "$MAPS_PATH/$NAME.map"
fi
# POI
if [ "$SKIP_POI_CREATION" != "true" ]; then
# modified by Junichi MORI, 2019/5/1
#CMD="$OSMOSIS_HOME/bin/osmosis --rb file=$WORK_PATH/$NAME-latest.osm.pbf \
# --pw file=$WORK_PATH/$NAME.poi \
# progress-logs=$PROGRESS_LOGS"
CMD="$OSMOSIS_HOME/bin/osmosis --rb file=$WORK_PATH/$BOUNDED_PBF \
--pw file=$WORK_PATH/$NAME.poi \
progress-logs=$PROGRESS_LOGS"
[ $POI_TAG_CONF_FILE ] && CMD="$CMD tag-conf-file=$POI_TAG_CONF_FILE"
echo $CMD
$CMD || exit 1
mv "$WORK_PATH/$NAME.poi" "$POIS_PATH/$NAME.poi"
fi
# Post-process
#rm -rf "$WORK_PATH"
#!/bin/bash
#
# upload map and ghz file to S3 bucket
#
# Junichi MORI, 2019/5/2
#
WORK_DIR=/home/osm/work/offlinemap
SRC_DIR=$WORK_DIR/result
MAP_FILE=ise.map
GHZ_FILE=ise.ghz
TS_FILE=map_timestamp
BUCKET="s3://バケット名"
IAM_PROFILE=ユーザー名
cd $SRC_DIR
if [ -f $MAP_FILE ]; then
aws s3 cp $MAP_FILE $BUCKET --profile=$IAM_PROFILE
RES=$?
if [ $RES != 0 ]; then
echo $MAP_FILE" ファイルのアップロードに失敗しました。"
exit $RES
fi
else
echo $MAP_FILE" ファイルがありません。"
fi
if [ -f $TS_FILE ]; then
aws s3 cp $TS_FILE $BUCKET --profile=$IAM_PROFILE
RES=$?
if [ $RES != 0 ]; then
echo $TS_FILE" ファイルのアップロードに失敗しました。"
exit $RES
fi
else
echo $TS_FILE" ファイルがありません。"
fi
if [ -f $GHZ_FILE ]; then
aws s3 cp $GHZ_FILE $BUCKET --profile=$IAM_PROFILE
RES=$?
if [ $RES != 0 ]; then
echo $GHZ_FILE" ファイルのアップロードに失敗しました。"
exit $RES
fi
else
echo $GHZ_FILE" ファイルがありません。"
fi
exit 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.