mapsforge map creation

doc.md

You'll need /data and /install folders, if this is not desired you'll need to change the script variables for the paths

1. install osmosis 0.43.1 http://bretth.dev.openstreetmap.org/osmosis-build/ to /install/osmosis
2. do git clone https://github.com/graphhopper/graphhopper /install/graphhopper_mapscreation then do git checkout 0.3
3. try cd /install/graphhopper_mapscreation; ./graphhopper.sh web europe_germany_berlin.pbf, this should not fail
4. create a new directory which should be accessible from outside /data/public/maps/ and contains subdirectory '0.4'
5. create /data/osm and /data/mapsforge where pbf and map files will be stored
6. install java to /install/java so that one can execute /install/java/bin/java
7. copy mapsforge writer 0.4.3 to /home//.openstreetmap/osmosis/plugins/mapsforge-map-writer-0.4.3-jar-with-dependencies.jar from http://ci.mapsforge.org/job/release-0.4.3/ or from maven central
8. include as many map names for de_maps, us_maps, eu_maps as possible. The naming scheme is derived from geofabrik as we download the pbfs from there: http://download.geofabrik.de/
9. test script via ./generate_maps.sh and then create a crontab with the following entry for weekly running: 0 18 */7 * * /home/peter/generate-maps.sh

generate-maps.sh

#!/bin/bash

# what this script does:
# * graphhopper will grab the pbf and create the necessary routing files
# * osmosis will create the map via the mapsforge writer plugin and uses the previously downloaded pbfs
# * finally it will be zipped and 'published'
#
# to force that the latest OSM is used one has to remove the pbf, graphhopper and map files or use 'true' for FORCE_UPDATE, DO_GH, DO_MF
# adding a new area is simple: just add to the variable and rerun existing maps and graphhopper files should be skipped and only the new map needs work

SCRIPT_HOME=$(dirname $0)

# TODO LATER current android version expects vehicles in map file too :(  
# also ".CAR-gh" will be truncated and if loaded again not found
# VEHICLES=CAR
VERSION=0.3
FORCE_UPDATE=true
DO_GH=true
DO_MF=true

LOG=/home/peter/generate-maps.sh.log
GH_HOME=/install/graphhopper_mapscreation
GH=$GH_HOME/graphhopper.sh
OSMOSIS_DIR=/install/osmosis
OSMOSIS=$OSMOSIS_DIR/bin/osmosis
OSM_DIR=/data/osm
MF_DIR=/data/mapsforge
MAPS_DIR=/data/public/maps/$VERSION
export JAVA_HOME=/install/java

# for osmosis
export JAVACMD=$JAVA_HOME/bin/java

de_maps="berlin bayern sachsen"
eu_maps="germany"
us_maps="us_new-york"

#de_maps="berlin bayern sachsen hamburg baden-wuerttemberg"
#eu_maps="spain germany france poland czech-republic \
#         france_ile-de-france france_provence-alpes-cote-d-azur france_rhone-alpes \
#         great-britain great-britain_england great-britain_wales great-britain_scotland"
#us_maps="us-midwest us-northeast us-pacific us-south us-west us_new-york us_california us_florida us_washington us_virginia"
#maps="asia_israel-and-palestine asia_japan"

for map in $de_maps; do
  maps="$maps europe_germany_$map"
done

for map in $eu_maps; do
  maps="$maps europe_$map"
done  

for map in $us_maps; do
  maps="$maps north-america_$map"
done
  
rm $LOG  >/dev/null 2>&1

if [ "x$FORCE_UPDATE" = "xtrue" ]; then
  echo "FORCED UPDATE will remove all maps in $MAPS_DIR. Press CTRL+C to cancel"
  read -e    
   
  rm -rf $OSM_DIR/*-gh >/dev/null 2>&1    
  rm $OSM_DIR/*.pbf >/dev/null 2>&1
  rm $MF_DIR/*.map  >/dev/null 2>&1
  rm $MAPS_DIR/*.ghz  >/dev/null 2>&1
fi

echo `date`" starting downloading pbf ..."
cd $OSM_DIR
for map in $maps; do
  OSM_FILE=$map.pbf
  # only download if not existing
  if [[ -s "$OSM_FILE" ]]; then
    echo "skip downloading $OSM_FILE"   
    continue 
  fi
  
  LINK=`echo $map | tr '_' '/'`   
  LINK="http://download.geofabrik.de/$LINK-latest.osm.pbf"
  echo "downloading $LINK"
  wget -S -nv -O $OSM_FILE $LINK
  size=`stat -c %s $OSM_FILE`
  echo $size
  ls $OSM_FILE
  if [[ ! -s "$OSM_FILE" ]] || [[ $size -lt 100 ]]; then
    echo "problem while downloading $OSM_FILE"
    exit
  fi
  
  # recreate maps, gh and ghz
  rm $MAPS_DIR/$map.ghz >/dev/null 2>&1
  rm $OSM_DIR/$map-gh >/dev/null 2>&1    
  rm $MF_DIR/$map.map >/dev/null 2>&1         
done

if [ "x$DO_GH" = "xtrue" ]; then 
  export JAVA_OPTS="-XX:PermSize=60m -XX:MaxPermSize=60m -Xmx4000m -Xms4000m"
  echo `date`" generate graphhopper files"
  # TODO create two subshells two execute graphhopper+mapsforge in parallel
  cd $GH_HOME
  for map in $maps; do
    from_map=$OSM_DIR/$map.pbf
    # gh_folder=$OSM_DIR/$map.$VEHICLES-gh/
    gh_folder=$OSM_DIR/$map-gh/

    if [[ -s "$gh_folder/nodes" ]]; then
      echo "already there: $gh_folder"
      continue 
    fi
    
    echo "generate $gh_folder"
    
    
    $GH import $from_map <<< '' >> $LOG 2>&1
    size=`stat -c %s $gh_folder/nodes`
    if [[ $? -ne 0 ]] || [[ $size -lt 100 ]]; then    
      echo "graphhopper problem for map $map" 
      exit
    fi     
    
    # force recreation of ghz
    rm $MAPS_DIR/$map.ghz >/dev/null 2>&1        
  done
fi

if [ "x$DO_MF" = "xtrue" ]; then

  echo `date`" create mapsforge maps ..."

  # if this increases too much we can specify type=hd
  # 5gb is not sufficient for bavaria!?
  export JAVACMD_OPTIONS="-Xmx4000m -Xms4000m -server"

  # make osmosis find the plugins/ folder
  cd $OSMOSIS_DIR
  for map in $maps; do
    to_map=$MF_DIR/$map.map

    if [[ -s $to_map ]]; then
      echo "already there: $to_map"
      continue
    fi
    
    echo `date`" create $to_map"
    
    $OSMOSIS --rb file=$OSM_DIR/$map.pbf --mapfile-writer file=$to_map type=hd preferred-language=en >> $LOG 2>&1
    if [[ $? -ne 0 ]]; then
       echo "mapsforge problem for map $map"
       exit
    fi
    
    # force recreation of ghz
    rm $MAPS_DIR/$map.ghz >/dev/null 2>&1        
  done
fi

echo `date`" create ghz files ..."
  
cd $OSM_DIR
for map in $maps; do
  # map_ghz=$MAPS_DIR/$map.$VEHICLES.ghz  
  map_ghz=$MAPS_DIR/$map.ghz
  
  if [[ -s $map_ghz ]]; then
     echo "already there: $map_ghz"
     continue
  fi                   
  
  # tmp_check="$map.$VEHICLES-gh"
  tmp_check="$map-gh"
  if [[ ! -e "$tmp_check/edges" ]]; then
     echo "ERROR no content in graphhopper folder $tmp_check"
     exit
  fi
  
  echo `date`" create $map_ghz"
  pushd $tmp_check > /dev/null
  
  cp $MF_DIR/$map.map .
  # cp $MF_DIR/$map.map $map.$VEHICLES.map
  zip -r $map_ghz * >> $LOG 2>&1
  popd > /dev/null
done
  
echo `date`" finished"