オフラインマップ生成用のスクリプト類

README.md

ブログ記事

避難所検索＠伊勢 オフライン用 map, ghz ファイルの自動生成

で使っているファイル群

• map-creator : mapsforge-creator の map-creator を修正したやつ
• doMapCreator : map-creator 実行スクリプト
• doGh : graphhopper 実行およびghzファイルに圧縮スクリプト
• uploadS3 : S3 へのアップロードスクリプト
• createOfflineMap : 全体制御スクリプト

更新情報

2021/4/14

OpenStreetMap のオフラインマップ作成時の不具合の修正 (3/3)

による変更を反映した。 なお、multipolygon2polygons.py については、下記の gist を元に入力ファイル名を変更したものになる。

https://gist.github.com/anonymous/735330

createOfflineMap

#!/bin/bash
#
# create files for offline map.
#
# Junichi MORI, 2019/5/1

function log()
{
    echo `date +"%F %T %Z"`": "$*
}

BIN_DIR=/home/osm/bin/offlinemap

MAP_SCRIPT=doMapCreator
GH_SCRIPT=doGh
UPLOAD_SCRIPT=uploadS3

WORK_DIR=/home/osm/work/offlinemap
MAP_DIR="$WORK_DIR/maps/v4/asia/japan"
MAP_FILE=kansai.map
MAP_OUT=ise.map

GHZ_DIR=$WORK_DIR/data/asia/japan/kansai/kansai-gh
GHZ_FILE=kansai.ghz
GHZ_OUT=ise.ghz

TS_FILE=map_timestamp

RESULT_DIR=$WORK_DIR/result

### main process start ###

log "start createOfflineMap script."

if [ ! -f $BIN_DIR/$MAP_SCRIPT ]; then
    log "no "$MAP_SCRIPT" file"
    exit 1
fi
if [ ! -f $BIN_DIR/$GH_SCRIPT ]; then
    log "no "$GH_SCRIPT" file"
    exit 1
fi

# 1. craete map file
log create map file
log $BIN_DIR/$MAP_SCRIPT
$BIN_DIR/$MAP_SCRIPT

if [ $? != 0 ]; then
    log "error in map-creation"
    exit 1
fi

# 2. craete ghz file
log create ghz file
log $BIN_DIR/$GH_SCRIPT
$BIN_DIR/$GH_SCRIPT
if [ $? != 0 ]; then
    log "error in graphhopper"
    exit 1
fi


# 3. copy created file 
log copy created files

TS_OLD=`cat $RESULT_DIR/$TS_FILE`
# file copy
if [ -f $MAP_DIR/$MAP_FILE ]; then
    cp -p $MAP_DIR/$MAP_FILE $RESULT_DIR/$MAP_OUT

    # timestamp file
    date --reference=$RESULT_DIR/$MAP_OUT --iso-8601=seconds > $RESULT_DIR/$TS_FILE
fi
if [ -f $GHZ_DIR/$GHZ_FILE ]; then
    cp -p $GHZ_DIR/$GHZ_FILE $RESULT_DIR/$GHZ_OUT
fi


# 4. upload to S3 bucket
if [ $TS_OLD == `cat $RESULT_DIR/$TS_FILE` ]; then
    log "map file timestamp is not updated. so skip S3 upload."
else
    log upload S3 bucket
    log $BIN_DIR/$UPLOAD_SCRIPT
    $BIN_DIR/$UPLOAD_SCRIPT
    if [ $? != 0 ]; then
        log "error to upload S3"
        exit 1
    fi
fi
log "finished createOfflineMap script."
exit 0

doGh

#!/bin/bash
#
# do graphhopper to create ghz file
#
# written by Junichi MORI
# 2019/4/23
#

function log()
{
    echo `date +"%F %T %Z"`": "$*
}

GH_BIN=$HOME/bin/graphhopper

export MAVEN_HOME=$GH_BIN/maven

WORK_DIR="/home/osm/work/offlinemap"
PBF_DIR="$WORK_DIR/data/asia/japan/kansai"
PBF_FILE="$PBF_DIR/kansai.pbf"

GH_DIR="$PBF_DIR/`basename $PBF_DIR`""-gh"
GHZ_FILE="`basename $PBF_DIR`"".ghz"

log target output dir : $GH_DIR
log target output file: $GHZ_FILE

# 1. preparation
if [ -d $GH_DIR ]; then
  rm -rf $GH_DIR
fi

# 2. create

cd $GH_BIN

# for 0.12.0
log $GH_BIN/graphhopper.sh --action import --input $PBF_FILE --config $WORK_DIR/gh_config.yml
./graphhopper.sh --action import --input $PBF_FILE --config $WORK_DIR/gh_config.yml

if [ $? != 0 ]; then
  log "graphhopper import failed."
  exit 1
fi

# 3. zip
cd $GH_DIR
pwd
log zip $GHZ_FILE ./*
zip $GHZ_FILE ./*
exit 0

doMapCreator

#!/bin/bash
#
# do mapsforge/map-creator
#
# written by Junichi MORI
# 2019/4/23
#

BIN_DIR="/home/osm/bin/mapsforge-creator"
MAP_CREATOR="map-creator"

$BIN_DIR/$MAP_CREATOR asia/japan/kansai ram ja ise.poly
exit $?

map-creator

#!/bin/bash

# Mapsforge map creation (with coastlines)
# using OpenStreetMap data from Geofabrik
#
# https://github.com/mapsforge/mapsforge/blob/master/docs/MapCreation.md
#
# Written by devemux86

# Configuration

# http://wiki.openstreetmap.org/wiki/Osmosis
#[ $OSMOSIS_HOME ] || OSMOSIS_HOME="$HOME/programs/osmosis"
[ $OSMOSIS_HOME ] || OSMOSIS_HOME="$HOME/bin/osmosis-0.47"

# add by Junichi MORI, 2019/5/1
[ $WORK_HOME ] || WORK_HOME="$HOME/work/offlinemap"

#[ $DATA_PATH ] || DATA_PATH="$HOME/mapsforge/data"
[ $DATA_PATH ] || DATA_PATH="$WORK_HOME/data"

#[ $MAPS_PATH ] || MAPS_PATH="$HOME/mapsforge/maps"
[ $MAPS_PATH ] || MAPS_PATH="$WORK_HOME/maps"

#[ $POIS_PATH ] || POIS_PATH="$HOME/mapsforge/pois"
[ $POIS_PATH ] || POIS_PATH="$WORK_HOME/pois"

[ $PROGRESS_LOGS ] || PROGRESS_LOGS="true"

[ $TAG_VALUES ] || TAG_VALUES="false"

[ $THREADS ] || THREADS="1"

#[ $DAYS ] || DAYS="30"
[ $DAYS ] || DAYS="27"

# added by Junichi MORI, 2019/5/1
SKIP_POI_CREATION="true"

# =========== DO NOT CHANGE AFTER THIS LINE. ===========================
# Below here is regular code, part of the file. This is not designed to
# be modified by users.
# ======================================================================

if [ $# -lt 2 ]; then
  #echo "Usage: $0 continent/country[/region] ram|hd [lang,...]"
  #echo "Example: $0 europe/germany/berlin ram en,de,fr,es"
  echo "Usage: $0 continent/country[/region] ram|hd [lang,...] [poly_file]"
  echo "Example: $0 europe/germany/berlin ram en,de,fr,es ise.poly"
  exit
fi

cd "$(dirname "$0")"

NAME="$(basename "$1")"

WORK_PATH="$DATA_PATH/$1"

if [ "$TAG_VALUES" = "true" ]; then
  MAPS_PATH="$MAPS_PATH/v5"
else
  [ $3 ] && MAPS_PATH="$MAPS_PATH/v4" || MAPS_PATH="$MAPS_PATH/v3"
fi
MAPS_PATH="$(dirname "$MAPS_PATH/$1")"

POIS_PATH="$(dirname "$POIS_PATH/$1")"

# check 2nd argument
if [ $2 != "ram" -a $2 != "hd" ]; then
  echo "2nd argument is invalid. must be ram or hd. : " $2
  exit
fi

# Check map date

if [ -f "$MAPS_PATH/$NAME.map" ]; then
  if [ $(find "$MAPS_PATH/$NAME.map" -mtime -$DAYS) ]; then
    echo "$MAPS_PATH/$NAME.map exists and is newer than $DAYS days."
    exit
  fi
fi

# Pre-process

rm -rf "$WORK_PATH"
mkdir -p "$WORK_PATH"

if [ "$SKIP_MAP_CREATION" != "true" ]; then
  mkdir -p "$MAPS_PATH"
fi

if [ "$SKIP_POI_CREATION" != "true" ]; then
  mkdir -p "$POIS_PATH"
fi

# Download land

if [ -f "$DATA_PATH/land-polygons-split-4326/land_polygons.shp" ] && [ $(find "$DATA_PATH/land-polygons-split-4326/land_polygons.shp" -mtime -$DAYS) ]; then
  echo "Land polygons exist and are newer than $DAYS days."
else
  echo "Downloading land polygons..."
  rm -rf "$DATA_PATH/land-polygons-split-4326"
  rm -f "$DATA_PATH/land-polygons-split-4326.zip"
  wget -nv -N -P "$DATA_PATH" https://osmdata.openstreetmap.de/download/land-polygons-split-4326.zip || exit 1
  unzip -oq "$DATA_PATH/land-polygons-split-4326.zip" -d "$DATA_PATH"
fi

# Download data

echo "Downloading $1..."
wget -nv -N -P "$WORK_PATH" https://download.geofabrik.de/$1-latest.osm.pbf || exit 1
wget -nv -N -P "$WORK_PATH" https://download.geofabrik.de/$1-latest.osm.pbf.md5 || exit 1
(cd "$WORK_PATH" && exec md5sum -c "$NAME-latest.osm.pbf.md5") || exit 1
wget -nv -N -P "$WORK_PATH" https://download.geofabrik.de/$1.poly || exit 1

# modified by Junichi MORI, 2019/5/1
# Bounds

[ $4 ] && POLY_FILE="$WORK_HOME/$4" || POLY_FILE="$WORK_PATH/$NAME.poly"
echo "Poly file is " $POLY_FILE

#BBOX=$(perl poly2bb.pl "$WORK_PATH/$NAME.poly")
BBOX=$(perl poly2bb.pl "$POLY_FILE")
BBOX=(${BBOX//,/ })
BOTTOM=${BBOX[0]}
LEFT=${BBOX[1]}
TOP=${BBOX[2]}
RIGHT=${BBOX[3]}

# Start position

# modified by Junichi MORI, 2019/5/1
#CENTER=$(perl poly2center.pl "$WORK_PATH/$NAME.poly")
CENTER=$(perl poly2center.pl "$POLY_FILE")
CENTER=(${CENTER//,/ })
LAT=${CENTER[0]}
LON=${CENTER[1]}

# added by Junichi MORI, 2019/5/1
# extract only bounding box
BOUNDED_PBF="$NAME.pbf"
echo $BOUNDED_PBF

CMD="$OSMOSIS_HOME/bin/osmosis --rb file=$WORK_PATH/$NAME-latest.osm.pbf \
                 --bb left=$LEFT right=$RIGHT top=$TOP bottom=$BOTTOM completeWays=yes \
                 --wb file=$WORK_PATH/$BOUNDED_PBF"
echo $CMD
$CMD

# Land

ogr2ogr -overwrite -progress -skipfailures -clipsrc $LEFT $BOTTOM $RIGHT $TOP "$WORK_PATH/land.shp" "$DATA_PATH/land-polygons-split-4326/land_polygons.shp"
# added by Junichi MORI, 2020/9/24
CURRENT=`pwd`
pushd $WORK_PATH
python $CURRENT/multipolygon2polygons.py
popd
# end of added
# modified by Junichi MORI, 2020/9/24
#python shape2osm.py -l "$WORK_PATH/land" "$WORK_PATH/land.shp"
python shape2osm.py -l "$WORK_PATH/land" "$WORK_PATH/polys.shp"

# Sea

cp sea.osm "$WORK_PATH"
sed -i "s/\$BOTTOM/$BOTTOM/g" "$WORK_PATH/sea.osm"
sed -i "s/\$LEFT/$LEFT/g" "$WORK_PATH/sea.osm"
sed -i "s/\$TOP/$TOP/g" "$WORK_PATH/sea.osm"
sed -i "s/\$RIGHT/$RIGHT/g" "$WORK_PATH/sea.osm"

# Merge

# modified by Junichi MORI, 2019/5/1
#CMD="$OSMOSIS_HOME/bin/osmosis --rb file=$WORK_PATH/$NAME-latest.osm.pbf \
#                               --rx file=$WORK_PATH/sea.osm --s --m"
CMD="$OSMOSIS_HOME/bin/osmosis --rb file=$WORK_PATH/$BOUNDED_PBF \
                               --rx file=$WORK_PATH/sea.osm --s --m"
for f in $WORK_PATH/land*.osm; do
  CMD="$CMD --rx file=$f --s --m"
done
CMD="$CMD --wb file=$WORK_PATH/merge.pbf omitmetadata=true"
echo $CMD
$CMD

# Map

if [ "$SKIP_MAP_CREATION" != "true" ]; then
  CMD="$OSMOSIS_HOME/bin/osmosis --rb file=$WORK_PATH/merge.pbf"
  [ $MAP_TRANSFORM_FILE ] && CMD="$CMD --tt file=$MAP_TRANSFORM_FILE"
  CMD="$CMD --mw file=$WORK_PATH/$NAME.map \
                 type=$2 \
                 bbox=$BOTTOM,$LEFT,$TOP,$RIGHT \
                 map-start-position=$LAT,$LON \
                 map-start-zoom=8 \
                 tag-values=$TAG_VALUES \
                 threads=$THREADS \
                 progress-logs=$PROGRESS_LOGS"
  [ $3 ] && CMD="$CMD preferred-languages=$3"
  [ $MAP_TAG_CONF_FILE ] && CMD="$CMD tag-conf-file=$MAP_TAG_CONF_FILE"
  echo $CMD
  $CMD || exit 1

  # Check map size

  if [ -f "$MAPS_PATH/$NAME.map" ]; then
    OLD_SIZE=$(wc -c < "$MAPS_PATH/$NAME.map")
    NEW_SIZE=$(wc -c < "$WORK_PATH/$NAME.map")
    if [ $NEW_SIZE -lt $(($OLD_SIZE * 70 / 100)) ]; then
      echo "$WORK_PATH/$NAME.map creation is significantly smaller."
      exit 1
    fi
  fi
  mv "$WORK_PATH/$NAME.map" "$MAPS_PATH/$NAME.map"
fi

# POI

if [ "$SKIP_POI_CREATION" != "true" ]; then
  # modified by Junichi MORI, 2019/5/1
  #CMD="$OSMOSIS_HOME/bin/osmosis --rb file=$WORK_PATH/$NAME-latest.osm.pbf \
  #                               --pw file=$WORK_PATH/$NAME.poi \
  #                                    progress-logs=$PROGRESS_LOGS"
  CMD="$OSMOSIS_HOME/bin/osmosis --rb file=$WORK_PATH/$BOUNDED_PBF \
                                 --pw file=$WORK_PATH/$NAME.poi \
                                      progress-logs=$PROGRESS_LOGS"
  [ $POI_TAG_CONF_FILE ] && CMD="$CMD tag-conf-file=$POI_TAG_CONF_FILE"
  echo $CMD
  $CMD || exit 1
  mv "$WORK_PATH/$NAME.poi" "$POIS_PATH/$NAME.poi"
fi

# Post-process

#rm -rf "$WORK_PATH"

multipolygon2polygons.py

import os, sys
from osgeo import ogr,gdal

def multipoly2poly(in_lyr, out_lyr):
    for in_feat in in_lyr:
        geom = in_feat.GetGeometryRef()
        if geom.GetGeometryName() == 'MULTIPOLYGON':
            for geom_part in geom:
                addPolygon(geom_part.ExportToWkb(), out_lyr)
        else:
            addPolygon(geom.ExportToWkb(), out_lyr)

def addPolygon(simplePolygon, out_lyr):
    featureDefn = out_lyr.GetLayerDefn()
    polygon = ogr.CreateGeometryFromWkb(simplePolygon)
    out_feat = ogr.Feature(featureDefn)
    out_feat.SetGeometry(polygon)
    out_lyr.CreateFeature(out_feat)
    print 'Polygon added.'


gdal.UseExceptions()
driver = ogr.GetDriverByName('ESRI Shapefile')
#in_ds = driver.Open('poly.shp', 0)
in_ds = driver.Open('land.shp', 0)
in_lyr = in_ds.GetLayer()
outputshp = 'polys.shp'
if os.path.exists(outputshp):
    driver.DeleteDataSource(outputshp)

out_ds = driver.CreateDataSource(outputshp)
out_lyr = out_ds.CreateLayer('polys', geom_type=ogr.wkbPolygon)
multipoly2poly(in_lyr, out_lyr)

uploadS3

#!/bin/bash
#
# upload map and ghz file to S3 bucket
#
# Junichi MORI, 2019/5/2
#

function log()
{
    echo `date +"%F %T %Z"`": "$*
}

WORK_DIR=/home/osm/work/offlinemap
SRC_DIR=$WORK_DIR/result

MAP_FILE=ise.map
GHZ_FILE=ise.ghz
TS_FILE=map_timestamp

BUCKET="s3://バケット名"
IAM_PROFILE=ユーザー名


cd $SRC_DIR

if [ -f $MAP_FILE ]; then
  aws s3 cp $MAP_FILE $BUCKET --profile=$IAM_PROFILE
  RES=$?
  if [ $RES != 0 ]; then
    log $MAP_FILE" ファイルのアップロードに失敗しました。"
    exit $RES
  fi
else
  log $MAP_FILE" ファイルがありません。"
fi

if [ -f $TS_FILE ]; then
  aws s3 cp $TS_FILE $BUCKET --profile=$IAM_PROFILE
  RES=$?
  if [ $RES != 0 ]; then
    log $TS_FILE" ファイルのアップロードに失敗しました。"
    exit $RES
  fi
else
  log $TS_FILE" ファイルがありません。"
fi

if [ -f $GHZ_FILE ]; then
  aws s3 cp $GHZ_FILE $BUCKET --profile=$IAM_PROFILE
  RES=$?
  if [ $RES != 0 ]; then
    log $GHZ_FILE" ファイルのアップロードに失敗しました。"
    exit $RES
  fi
else
  log $GHZ_FILE" ファイルがありません。"
fi
exit 0