Skip to content

Instantly share code, notes, and snippets.

@ramunasd
Created September 20, 2012 09:31
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save ramunasd/3754915 to your computer and use it in GitHub Desktop.
OSM latest planet file torrent create and update scripts
#!/bin/bash
#
# Downloads latest OpenStreetMap planet file and creates torrent for it.
# Based on http://osm-torrent.torres.voyager.hr/ scripts
#
# You should edit "WORKDIR=" line, and copy this script in your /etc/cron.daily
#
DEF_WORKDIR=. # you must change this, if nothing else...
DEF_FILE_TYPE=planet # "planet" or "pbfplanet" (or "changesets" for faster testing)
WGET_OPTIONS="--limit-rate=1000k" # if you want to speed limit wget of PLANET etc.
# those can be overriden from environment, for example:
# env FILE_TYPE=changesets EXPIRE_DAYS=30 WORKDIR=/tmp DATE=101201 create_new_planet_torrent.sh
############################################
# no user configurable parts below
############################################
DEBUG=${DEBUG:-0}
WORKDIR=${WORKDIR:-$DEF_WORKDIR}
FILE_TYPE=${FILE_TYPE:-$DEF_FILE_TYPE}
DATE=${DATE:-$(date --date '2 days ago' +%y%m%d)} # due to timezones and stuff, it seems we only see the planet when we're already two days ahead. compensate in order to get right planet name... This would not be needed if the script is run on OSMF servers themselves, and you'd have torrents immedaately instead of 48 hours delay....
cd "$WORKDIR" || exit 101
LICENSE="© OpenStreetMap contributors"
if [ "$FILE_TYPE" = "pbfplanet" ]
then
FILE_PLANET="planet-latest.osm.pbf"
URL_EXTRA_DIR="pbf/"
else
exit
fi
FILE_MD5="${FILE_PLANET}.md5"
FILE_TORRENT="${FILE_PLANET}.torrent"
URL_PLANET2="http://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/${URL_EXTRA_DIR}${FILE_PLANET}" # Germany, mirror planet.osm.org
URL_PLANET="http://planet.osm.org/${URL_EXTRA_DIR}${FILE_PLANET}" # webseed fallback, original site
URL_MD5="${URL_PLANET}.md5"
CHUNKSIZE=22 # 2^20=1MB, 2^22=4MB, etc. mktorrent 1.0 default ( 2^18=256kB) is too small for our ~15GB files
[ "$DEBUG" -gt 0 ] && echo "source=$URL_PLANET, md5=$URL_MD5, torrent=$FILE_TORRENT"
[ "$DEBUG" -gt 8 ] && exit 9
# abort new download if download currently in progress!
[ -f "$FILE_PLANET" ] && fuser -s "$FILE_PLANET" && exit 1
LAST=`stat -c %Y $FILE_MD5`
echo $LAST
wget -Nq "$WGET_OPTIONS" "$URL_MD5"
CURRENT=`stat -c %Y $FILE_MD5`
echo $CURRENT
RET=$?
[ "$DEBUG" -gt 1 ] && echo "wget $URL_MD5 -- return code $RET"
if [ $CURRENT -le $LAST ]
then
[ "$DEBUG" -gt 1 ] && echo "already latest planet file"
exit
fi
# remove torrent file earlier
rm $FILE_TORRENT
# retrieve new planet file name
FILE_PLANET=`cat $FILE_MD5 | awk '{print $2}'`
FILE_TORRENT="${FILE_PLANET}.torrent"
URL_PLANET="http://planet.osm.org/${URL_EXTRA_DIR}${FILE_PLANET}" # webseed fallback, original site
# download new planet (if remote file changed)
wget -qN -c "$WGET_OPTIONS" "$URL_PLANET"
RET=$?
[ "$DEBUG" -gt 1 ] && echo "wget $URL_PLANET -- return code $RET"
# exit silently if nothing downloaded
[ -f "$FILE_PLANET" ] || exit 0
# exit silently if planet file didn't change (torrent is newer than file)
[ "$FILE_TORRENT" -nt "$FILE_PLANET" ] && exit 0
if md5sum -c $FILE_MD5
then
echo checksum ok
else
echo checksum failed, aborting
rm $FILE_TORRENT
rm $FILE_MD5
exit 2
fi
[ "$DEBUG" -gt 1 ] && echo "running mktorrent to create $FILE_TORRENT"
# this is our full featured torrent file: redundant trackers, tcp+udp, ipv4+ipv6, webseed
mktorrent -l $CHUNKSIZE \
-c "$LICENSE" \
-a http://ipv4.tracker.osm-torrent.torres.voyager.hr/announce \
-a udp://tracker.ipv6tracker.org:80/announce,http://tracker.ipv6tracker.org:80/announce \
-a udp://tracker.publicbt.com:80/announce,http://tracker.publicbt.com:80/announce \
-a udp://tracker.ccc.de:80/announce,http://tracker.ccc.de/announce \
-a udp://tracker.openbittorrent.com:80/announce \
-w $URL_PLANET2 -w $URL_PLANET $FILE_PLANET -o ${FILE_TORRENT}
[ "$DEBUG" -gt 1 ] && echo "running post-process.d for $FILE_TORRENT"
# run scripts in post-process.d directory if any (we can use it to cache torrent on torrage.com, generate RSS, etc)
export DEBUG
if [ -d ./post-process.d ]
then
run-parts --verbose --arg="$FILE_TORRENT" ./post-process.d
fi
exit 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment