Skip to content

Instantly share code, notes, and snippets.

@dakcarto
Created August 22, 2012 18:18
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save dakcarto/024c98d41607b19bd8fd to your computer and use it in GitHub Desktop.
Save dakcarto/024c98d41607b19bd8fd to your computer and use it in GitHub Desktop.
Bash script for local auto-builds of QGIS on Mac OS X
#!/bin/bash
# -*- coding: utf-8 -*-
###############################################################################
# qgis-nighty-autobuild.sh
# (c) 2012 Larry Shaffer larrys <at> dakotacarto <dot> com
#
# Build Mac OS X QGIS.app locally from github.com QGIS repository.
# Keep a number of dmg archives of previous builds as backups.
#
# Archived builds can be run from disk images later. No need to reinstall.
#
# Tested on Mac OS X 10.7.4, built apps run on 10.7 and 10.8
#
# Requires same kyngchaos.com framework setup and build environment as per
# https://github.com/qgis/Quantum-GIS/blob/master/INSTALL
#
# Built using kyngchaos.com's GRASS.app and PostgreSQL/PostGIS
#
###############################################################################
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
###############################################################################
## Run settings
###############################################################################
# HOMEBREW_PREFIX
HB=/usr/local/osgeo4mac
# Kyngchaos supporting libs install
KYNG=/usr/local/kyng
# Git setup
GIT=${HB}/opt/git/bin/git
GITURL="git://github.com/qgis/Quantum-GIS.git"
# Working directory and output locations
USERDIR=$(echo ~)
QGISDIR="$USERDIR/QGIS"
GITHUBDIR="$QGISDIR/nightly"
QGISGITDIR="$GITHUBDIR/Quantum-GIS"
BUILDDIR="$GITHUBDIR/build"
TARGETDIR="QGIS_Apps"
APPTARGETDIR="$GITHUBDIR/$TARGETDIR"
APPTARGETNAME="QGIS_2.3-dev"
APPTARGET="$APPTARGETDIR/$APPTARGETNAME.app"
APPBUNDIDENT="org.qgis.qgis-dev"
BROWSER="QGIS Browser.app"
BROWSERLINK="QGIS_Browser(symlink).app"
NIGHTLYDIR="$GITHUBDIR/archive-nightly"
NIGHTLYTEMP="$GITHUBDIR/archive-temp"
NIGHTLYNAME="qgis-mac-nightly"
NIGHTLYOUT="$NIGHTLYDIR/$NIGHTLYNAME"
APPARCHIVES="$GITHUBDIR/archives"
CHECKSUMS="DMG-Checksums.txt"
UPLOADSYNCDIR="$NIGHTLYDIR/upload-sync"
NIGHTLYZIP="$NIGHTLYNAME"".zip"
ZIPCHECKSUM="$NIGHTLYNAME""_checksums.txt"
# Name for remote to pull from: qgisupstream
# Script will add 'qgisupstream' remote to your local repository, if missing.
# Path to git patch file to apply to branch after switching to it.
# This patch will be applied to branch, branch built, then patch reversed.
# This is useful for getting a branch to build if it won't, yet,
# but reverses the fix and leaves the branch clean after build.
# If QGISTESTS=TRUE, the patch will not be removed, so tests can recompile.
# In such a case, patch will have to be manually removed before next compile.
#GITPATCH="$GITHUBDIR/patches/pkgdir_revert_patch.diff"
GITPATCH=""
GITPULL=1
# Use maximum number of CPU cores for compiling QGIS.
CPUCORES=$(/usr/sbin/sysctl -n hw.ncpu)
#CPUCORES=4
BUILDQGIS=1
QGISDEBUG=1
BUILDTYPE="MinSizeRel"
MACBUNDLE="2"
if [ $QGISDEBUG -gt 0 ]; then
BUILDTYPE="RelWithDebInfo"
# TODO: test if lesser bundling setting helps debugging
MACBUNDLE="2"
fi
# Whether to clean (rm -R dir && mkdir) build dir before building
CLEANBUILD=0
QGISSERVER="TRUE"
QGISAPIDOCS="FALSE"
INSTALLQGIS=1
QUIETINSTALL=1
# CTest options.
QGISTESTS="TRUE"
# Run tests and upload results and app?
PROCESSNIGHTLY=1
QGISRUNTESTS=$PROCESSNIGHTLY
RUNEXPTESTS=1
ARCHIVEAPP=$PROCESSNIGHTLY
BACKUPOLDARCHIVE=1
KEEPARCHIVES=14
ZIPARCHIVE=$PROCESSNIGHTLY
UPLOAD=$PROCESSNIGHTLY
###############################################################################
## Functions
###############################################################################
REVPATCH () {
cd "$QGISGITDIR"
echo -e "\nReversing patch to branch..."
$GIT apply --reverse "$GITPATCH"
if [ $? -gt 0 ]; then
echo -e "\nERROR reversing patch to branch!"
exit 1
fi
}
###############################################################################
## Start Build
###############################################################################
echo -e "\n\n################################################################\n"
echo -e "QGIS Build @ $(date)"
###############################################################################
## Update QGIS Working Tree
###############################################################################
cd "$GITHUBDIR"
# check for existing repo dir
if [ ! -d "$QGISGITDIR" ]; then
echo -e "\nRepository missing, cloning 'master' branch...\n"
mkdir "$QGISGITDIR" && cd "$QGISGITDIR"
$GIT init
$GIT remote add -f -t master -m master qgisupstream "$GITURL"
$GIT merge qgisupstream
if [ $? -gt 0 ]; then
echo -e "\nERROR in cloning git repo 'master' branch!"
rm -R "$QGISGITDIR"
exit 1
fi
# rerun script after clone
echo -e "\nQGIS master branch cloned. Rerun script to build."
exit
else
cd "$QGISGITDIR"
echo -e "\nRepository exists, checking for 'qgisupstream' remote..."
if [ $($GIT remote | egrep -c '^qgisupstream$') -gt 0 ]; then
echo "found remote"
else
$GIT remote add qgisupstream "$GITURL"
if [ $? -gt 0 ]; then
echo -e "\nERROR adding remote qgisupstream to local repository!"
exit 1
fi
echo "added remote"
fi
if [ $GITPULL -gt 0 ]; then
# clean git repo of leftover files generated from tests
echo -e "\nCleaning and resetting repository working tree..."
$GIT clean -xdf
if [ $? -gt 0 ]; then
echo -e "\nERROR cleaning git repository!"
exit 1
fi
$GIT reset --hard HEAD
if [ $? -gt 0 ]; then
echo -e "\nERROR resetting git repository!"
exit 1
fi
# update build branch
echo -e "\nFetching 'qgisupstream' and pulling 'master' branch..."
$GIT fetch qgisupstream
if [ $? -gt 0 ]; then
echo -e "\nERROR fetching git repository!"
exit 1
fi
$GIT pull qgisupstream master
if [ $? -gt 0 ]; then
echo -e "\nERROR pulling qgisupstream git repo 'master' branch!"
exit 1
fi
fi
fi
if [ "$GITPATCH" != "" ]; then
cd "$QGISGITDIR"
echo -e "\nPatching branch, checking patch validity...\n"
$GIT apply --check "$GITPATCH"
if [ $? -gt 0 ]; then
echo -e "\nERROR patch cannot be applied to branch!"
exit 1
else
echo -e "Patch can be applied, applying..."
fi
$GIT apply "$GITPATCH"
if [ $? -gt 0 ]; then
echo -e "\nERROR applying patch to branch!"
exit 1
fi
fi
echo -e "\nUpdating app name to dev version..."
/usr/bin/sed -i '' -e "s/QGIS_APP_NAME \"QGIS\"/QGIS_APP_NAME \"$APPTARGETNAME\"/" CMakeLists.txt
if [ $? -gt 0 ]; then
echo -e "\nERROR updating app name to dev version!"
exit 1
fi
###############################################################################
## Build QGIS.app
###############################################################################
if [ $BUILDQGIS -gt 0 ]; then
if [ $CLEANBUILD -gt 0 ]; then
# remove any previous build dir
if [ -d "$BUILDDIR" ]; then
echo -e "\nRemoving existing build directory ..."
rm -R "$BUILDDIR"
fi
fi
if [ ! -d "$BUILDDIR" ]; then
# QGIS CMake setup supports out-of-source dir builds
# make build directory and configure
mkdir -p "$BUILDDIR"
fi
cd "$BUILDDIR"
# force clang
export CC=/usr/bin/clang
export CXX=/usr/bin/clang++
# specify exact PATH, so Homebrew stuff is skipped
export PATH=${KYNG}/bin:/Library/Frameworks/FreeType.framework/Programs:/Library/Frameworks/FFTW3.framework/Programs:/Library/Frameworks/GDAL.framework/Programs:/Library/Frameworks/FFTW3.framework/Programs:/Library/Frameworks/GEOS.framework/unix/bin:/Library/Frameworks/PROJ.framework/Programs:/Library/Frameworks/SQLite3.framework/Programs:/Library/Frameworks/UnixImageIO.framework/Programs:/Library/Frameworks/cairo.framework/Programs:/usr/bin:/bin:/usr/sbin:/sbin:/usr/X11/bin
echo -e "\nConfiguring build ...\n"
${HB}/bin/cmake -D CMAKE_INSTALL_PREFIX="$APPTARGETDIR" \
-D CMAKE_BUILD_TYPE="$BUILDTYPE" \
-D BUILDNAME="lion-intel64-clang" \
-D SITE="qgis.dakotacarto.com" \
-D GITCOMMAND=${GIT} \
-D WITH_ASTYLE=FALSE \
-D ENABLE_TESTS="$QGISTESTS" \
-D WITH_INTERNAL_SPATIALITE=FALSE \
-D WITH_PYSPATIALITE=FALSE \
-D POSTGRES_CONFIG=${HB}/opt/postgresql/bin/pg_config \
-D SPATIALINDEX_INCLUDE_DIR=${HB}/opt/spatialindex/include/spatialindex \
-D SPATIALINDEX_LIBRARY=${HB}/opt/spatialindex/lib/libspatialindex.dylib \
-D QWT_LIBRARY=${KYNG}/lib/libqwt.dylib \
-D QWT_INCLUDE_DIR=${KYNG}/include \
-D QSCINTILLA_LIBRARY=${KYNG}/lib/libqscintilla2.dylib \
-D QSCINTILLA_INCLUDE_DIR=${KYNG}/include/Qsci \
-D WITH_STAGED_PLUGINS=FALSE \
-D WITH_GLOBE=TRUE \
-D OSGEARTH_INCLUDE_DIR="/Library/Application Support/OpenSceneGraph/Headers" \
-D OSG_PLUGINS_PATH="/Library/Application Support/OpenSceneGraph/PlugIns" \
-D BISON_EXECUTABLE=${HB}/opt/bison/bin/bison \
-D WITH_MAPSERVER="$QGISSERVER" \
-D FCGI_INCLUDE_DIR=${HB}/include \
-D FCGI_LIBRARY=${HB}/lib/libfcgi.dylib \
-D WITH_CUSTOM_WIDGETS=FALSE \
-D WITH_APIDOC="$QGISAPIDOCS" \
-D QGIS_MACAPP_BUNDLE="$MACBUNDLE" \
"$QGISGITDIR"
# /usr/bin/bbedit CMakeCache.txt && exit 0
if [ $? -gt 0 ]; then
echo -e "\nERROR configuring QGIS build!"
if [ "$GITPATCH" != "" ]; then REVPATCH; fi
exit 1
fi
#Build and install target QGIS.app
echo -e "\nMaking build ...\n"
make -j $CPUCORES
if [ $? -gt 0 ]; then
echo -e "\nERROR building QGIS!"
if [ "$GITPATCH" != "" ]; then REVPATCH; fi
exit 1
fi
if [ $QGISRUNTESTS -gt 0 ]; then
echo -e "\nRunning tests in 5 seconds..."
sleep 5
cd "$BUILDDIR"
if [ $RUNEXPTESTS -gt 0 ]; then
make -j 8 Experimental
else
make -j 8 test
fi
fi
if [ $INSTALLQGIS -gt 0 ]; then
if [ ! -d "$APPTARGETDIR" ]; then
mkdir -p "$APPTARGETDIR"
fi
# remove last app built if current repo built
if [ -d "$APPTARGET" ]; then
echo -e "\nRemoving previously installed app ..."
rm -R "$APPTARGET"
fi
echo -e "\nInstalling build to target app bundle...\n"
# only show errors on install?
if [ $QUIETINSTALL -gt 0 ]; then
make install 1>/dev/null
else
make install
fi
if [ $? -gt 0 ]; then
echo -e "\nERROR installing QGIS!"
if [ "$GITPATCH" != "" ]; then REVPATCH; fi
exit 1
fi
echo -e "\nUpdating app bundle identifier..."
/usr/libexec/PlistBuddy -c "Set :CFBundleIdentifier $APPBUNDIDENT" \
"$APPTARGET/Contents/Info.plist"
if [ $? -gt 0 ]; then
echo -e "\nERROR updating bundle identifier!"
if [ "$GITPATCH" != "" ]; then REVPATCH; fi
exit 1
fi
# echo -e "\nUpdating pyuic path..."
# # sed -E s, /.*/PyQt4/uic/pyuic.py, ${QDATADIR}/python/PyQt4/uic/pyuic.py,"
# # $(/usr/bin/dirname "$0")/../../Resources/python/PyQt4/uic/pyuic.py
# /usr/bin/sed -i '' -E "s, /.*/PyQt4/uic/pyuic.py, \"\$(/usr/bin/dirname \"\$0\")/../../Resources/python/PyQt4/uic/pyuic.py\"," "$APPTARGET/Contents/MacOS/bin/pyuic4"
# if [ $? -gt 0 ]; then
# echo -e "\nERROR updating app name to dev version!"
# # if [ "$GITPATCH" != "" ]; then REVPATCH; fi
# # exit 1
# fi
# remove build alias to QGIS Broswer
echo -e "\nRemoving browser app alias, in favor of symlink for archive..."
/usr/bin/find "$APPTARGETDIR" -type f -name "$BROWSER*" -delete
# make link to QGIS Browser.app
# link is better than alias, when launching from dmg
if [ -L "$APPTARGETDIR/$BROWSERLINK" ]; then
echo -e "\nRemoving previous browser app link ..."
rm "$APPTARGETDIR/$BROWSERLINK"
fi
cd "$APPTARGETDIR"
echo -e "\nCreating symbolic link to QGIS Browser.app ..."
ln -s "$APPTARGETNAME.app/Contents/MacOS/bin/$BROWSER" "./$BROWSERLINK"
if [ $? -gt 0 ]; then
echo -e "\nERROR creating link!"
fi
# if [ ! -L "Applications" ]; then
# echo -e "\nCreating symbolic link to /Applications ..."
#
# ln -s "/Applications" "./Applications"
#
# if [ $? -gt 0 ]; then
# echo -e "\nERROR creating link!"
# fi
# fi
# make sure the app bundle's components are user-writable, so it can be deleted later
# homewbrew's binaries, etc. are installed non-writable
chmod -R u+w "$APPTARGETNAME.app"
fi
fi
###############################################################################
## Reset repository
###############################################################################
# Reverse any patch file applied to current or master branch.
if [ "$GITPATCH" != "" ]; then
REVPATCH
fi
###############################################################################
## Archive QGIS.app and Backup to DMG Archives
###############################################################################
if [ $ARCHIVEAPP -gt 0 ]; then
sleep 5
# make sure dir structure exists
if [ ! -d "$NIGHTLYOUT" ]; then
mkdir -p "$NIGHTLYOUT"
fi
# congifure archive file
cd "$QGISGITDIR"
# this creates a short sha hash for last commit on current local branch built
REVHASH=$($GIT log --no-color --pretty=format:%H master^..master)
REVSHORTHASH=$(echo ${REVHASH:0:7})
REVMAJOR=$(grep 'SET(CPACK_PACKAGE_VERSION_MAJOR' CMakeLists.txt | egrep -o '[0-9]+')
REVMINOR=$(grep 'SET(CPACK_PACKAGE_VERSION_MINOR' CMakeLists.txt | egrep -o '[0-9]+')
REVPATCH=$(grep 'SET(CPACK_PACKAGE_VERSION_PATCH' CMakeLists.txt | egrep -o '[0-9]+')
VERNUM="${REVMAJOR}-${REVMINOR}-${REVPATCH}"
TODAY=$(date +"%Y-%m-%d")
APPNAME="QGIS_${VERNUM}_${TODAY}_${REVSHORTHASH}"
APPARCHIVE="$APPNAME.dmg"
TEMPARCHIVE="$APPNAME.sparseimage"
if [ $BACKUPOLDARCHIVE -gt 0 ]; then
echo -e "\nBacking up previous archive..."
# make sure dir structure exists
if [ ! -d "$APPARCHIVES" ]; then
mkdir -p "$APPARCHIVES"
fi
/usr/bin/find "$NIGHTLYOUT" -type f -name "QGIS_*.dmg" -exec mv {} "$APPARCHIVES/" \;
cd "$APPARCHIVES"
# prune dmg archives
echo -e "\nPruning archives..."
CURFILE=1
for f in $(ls -1t QGIS*); do
# add extra file for checksums
if [ $CURFILE -gt $KEEPARCHIVES ]; then
rm "$APPARCHIVES/$f"
fi
let CURFILE=CURFILE+1
done
# generate checksums file
echo -e "\nGenerating checksums file..."
if [ -e "$CHECKSUMS" ]; then
echo "" > "$CHECKSUMS"
else
touch "$CHECKSUMS"
fi
for f in $(ls -1t QGIS*); do
/usr/bin/openssl md5 "$f" >> "$CHECKSUMS"
/usr/bin/openssl sha1 "$f" >> "$CHECKSUMS"
echo "" >> "$CHECKSUMS"
done
else
/usr/bin/find "$NIGHTLYOUT" -type f -name "QGIS_*.dmg" -delete
fi
# archive built app to compressed dmg
echo -e "\nArchiving $TARGETDIR target directory..."
cd "$GITHUBDIR"
echo -e "\nDeleting any old temp archives..."
/usr/bin/find "$NIGHTLYTEMP" -type f -name "QGIS_*.sparseimage" -delete
# create dmg archive
echo -e "\nCreating $TEMPARCHIVE..."
hdiutil create -type SPARSE -size 720m -fs HFS+ -volname "$TARGETDIR" "$NIGHTLYTEMP/$TEMPARCHIVE"
if [ $? -gt 0 ]; then
echo -e "\nERROR creating $TEMPARCHIVE!"
exit 1
fi
echo -e "\nMounting $TEMPARCHIVE..."
hdiutil attach "$NIGHTLYTEMP/$TEMPARCHIVE"
if [ ! -d "/Volumes/$TARGETDIR" ]; then
echo -e "\nERROR mounting $TEMPARCHIVE to /Volumes/$TARGETDIR!"
exit 1
fi
echo -e "\nDitto-ing to /Volumes/$TARGETDIR..."
ditto -X --norsrc --nocache "$APPTARGETDIR" "/Volumes/$TARGETDIR"
if [ $? -gt 0 ]; then
echo -e "\nERROR ditto-ing $TARGETDIR to /Volumes/$TARGETDIR!"
exit 1
fi
echo -e "\nDetaching $TEMPARCHIVE (/Volumes/$TARGETDIR)..."
hdiutil detach "/Volumes/$TARGETDIR"
if [ $? -gt 0 ]; then
echo -e "\nERROR detaching /Volumes/$TARGETDIR!"
exit 1
fi
echo -e "\nCompacting $TEMPARCHIVE..."
hdiutil compact "$NIGHTLYTEMP/$TEMPARCHIVE"
if [ $? -gt 0 ]; then
echo -e "\nERROR compacting $TEMPARCHIVE!"
exit 1
fi
echo -e "\nConverting $TEMPARCHIVE to bzip2-compressed $APPARCHIVE..."
hdiutil convert "$NIGHTLYTEMP/$TEMPARCHIVE" -format UDBZ -o "$NIGHTLYOUT/$APPARCHIVE"
if [ $? -gt 0 ]; then
echo -e "\nERROR converting $TEMPARCHIVE!"
exit 1
fi
echo -e "\nDeleting any old temp archives..."
/usr/bin/find "$NIGHTLYTEMP" -type f -name "QGIS_*.sparseimage" -delete
fi
if [ $ZIPARCHIVE -gt 0 ]; then
# make sure dir structure exists
if [ ! -d "$UPLOADSYNCDIR" ]; then
mkdir -p "$UPLOADSYNCDIR"
fi
echo -e "\nClearing out upload sync directory..."
/usr/bin/find "$UPLOADSYNCDIR" -type f -delete
sleep 3
echo -e "\nGenerating ZIP archive..."
cd "$NIGHTLYDIR"
/usr/bin/zip -rq -9 "$NIGHTLYZIP" "$NIGHTLYNAME"
if [ $? -gt 0 ]; then
echo -e "\nERROR zipping QGIS archive!"
exit 1
fi
sleep 3
mv "$NIGHTLYZIP" "$UPLOADSYNCDIR"
# generate checksums file
echo -e "\nGenerating info and checksums file for ZIP..."
# get last commit as short info (sans email)
cd "$QGISGITDIR"
LASTCOMMIT=$($GIT show --format="Latest commit to master branch: %h%n'%s'%nby %an, %ar" --name-status)
cd "$UPLOADSYNCDIR"
echo -e "'$APPNAME' ZIP Archive Checksums\n" >> "$ZIPCHECKSUM"
/usr/bin/openssl md5 "$NIGHTLYZIP" >> "$ZIPCHECKSUM"
/usr/bin/openssl sha1 "$NIGHTLYZIP" >> "$ZIPCHECKSUM"
echo -e "\n$LASTCOMMIT\n" >> "$ZIPCHECKSUM"
fi
if [ $UPLOAD -gt 0 ]; then
if [ -d "$UPLOADSYNCDIR" ]; then
echo -e "\nUploading ZIP archive and checksums..."
cd "$UPLOADSYNCDIR"
/usr/local/osgeo4mac/bin/s3cmd put --acl-public \
--add-header "Cache-Control:no-cache" \
--mime-type "application/zip" \
"$NIGHTLYZIP" "s3://qgis.dakotacarto.com/nightly/$NIGHTLYZIP"
if [ $? -gt 0 ]; then
echo -e "\nERROR uploading QGIS archive!"
exit 1
fi
/usr/local/osgeo4mac/bin/s3cmd put --acl-public \
--add-header "Cache-Control:no-cache" \
--mime-type "text/plain" \
"$ZIPCHECKSUM" "s3://qgis.dakotacarto.com/nightly/$ZIPCHECKSUM"
if [ $? -gt 0 ]; then
echo -e "\nERROR uploading QGIS archive checksums!"
exit 1
fi
# reversed so archive uploads before checksums is changed
# requires libmagic for python-magic to guess mime types
# for f in $(ls -r1t *); do
# /usr/local/bin/s3cmd put --add-header "Cache-Control:no-cache" \
# -P --guess-mime-type "$f" "s3://qgis.dakotacarto.com/nightly/$f"
# if [ $? -gt 0 ]; then
# echo -e "\nERROR uploading QGIS archive or checksums!"
# exit 1
# fi
# done
fi
fi
echo -e "\nQGIS Build @ $(date) Finished"
exit 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment