shell-stuff.sh

#!/bin/bash

# stop on errors and undeclared variables
set -e -u -o pipefail

err() {
  echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $@" >&2
}

and in general follow these:
http://www.kfirlavi.com/blog/2012/11/14/defensive-bash-programming/
and http://www.davidpashley.com/articles/writing-robust-shell-scripts/
and http://mywiki.wooledge.org/BashPitfalls
and http://devmanual.gentoo.org/tools-reference/bash/
and maybe http://google-styleguide.googlecode.com/svn/trunk/shell.xml
...
more literature:
http://unix.stackexchange.com/questions/23026/how-can-i-get-bash-to-exit-on-backtick-failure-in-a-similar-way-to-pipefail
http://www.riedquat.de/blog/2011-03-06-01
http://www.gnu.org/software/bash/manual/bashref.html#The-Set-Builtin
http://redsymbol.net/articles/unofficial-bash-strict-mode/

For reference: SSH debugging

First export these vars to make the next part copy-pastable:

export PORT=2222
export USER="me"
export HOST="example.com"
export PRIVATE_KEY="~/.ssh/id_rsa"

Test an SFTP client:
sftp -vvv -o Port="${PORT}" -o IdentityFile="${PRIVATE_KEY}" "${USER}@${HOST}"

Test an SSH server (no deamon, intensive logging to stderr):
/usr/sbin/sshd -ddde -o Port="${PORT}"

Also SSH server, a little more serious:
strace /usr/sbin/sshd -D -q -e -o Port="${PORT}"

some useful functions

# SSH tunnel to remote MySQL; call as "sshmysql SSH_LOGIN@HOST"
alias sshmysql='ssh -L 3306:localhost:3306 -N'

# Recursively download a website with all assets; call as "download-website http://SITE.URL"
alias download-website='wget --mirror --page-requisites --html-extension --convert-links'

# serve current directory and open it in a browser (port may be given, default is 8080)
servecd() {
  python -m SimpleHTTPServer ${1:-8080} &
  open http://localhost:${1:-8080}
}

# open source of python module in editor
pysource() { subl -w $(python -c "import ${1}; import inspect; print inspect.getsourcefile(${1})"); }

# OS X specific

# man page as pdf
pman() { man -t "${1}" | open -f -a /Applications/Preview.app; }

# sometimes my Macbook imagines there's a CD when there's not
alias ejectcd='drutil tray eject'

# Sublime Text
export EDITOR_BIN='/Applications/Sublime\ Text.app/Contents/SharedSupport/bin/subl'
export EDITOR="$EDITOR_BIN -w"
export VISUAL=$EDITOR
alias subl="$EDITOR_BIN"

All cronjobs on a root accessible computer to markdown (call as lscronjobs HOSTNAME):

lscronjobs() {
    echo "## SERVER $1"
    ssh -o ConnectTimeout=1 -o ConnectionAttempts=2 "root@$1" 'bash -s' <<-'LS_CRONS'
        set -u -o pipefail
        for f in /etc/crontab /etc/cron.d/*
        do
            [ -f "$f" ] \
                && echo -e "\n### FILE $f\n" \
                && sed -e '/^$/d' -e 's/^/    /' < "$f" -e '/^[ \t#]+$/d'
        done
        for u in $(getent passwd | cut -f1 -d:)
        do
            crontab -u "$u" -l >& /dev/null \
                && echo -e "\n### USER $u\n" \
                && ( \
                    crontab -u "$u" -l 2> /dev/null \
                    | sed -e '/^$/d' -e 's/^/    /' -e '/^[ \t#]+$/d' )
        done
    LS_CRONS
    echo
}

Print a help text for environment variables used in the current file.
The variables must have a postfix comment starting with #@@ (Bash 4):

used_env() {
  echo "Environment variables influencing ${BASH_SOURCE}:"
  while IFS='' read -r LINE ; do
    # delete all before double-@; inject "" so the line is not matched itself
    local HELPTEXT="${LINE##*@""@}"
    if [[ "${HELPTEXT}" == "${LINE}" ]]; then
      continue
    fi
    local NAME="${LINE%%=*}"
    local CURRENT_VALUE="${!NAME}"
    echo -n -e "  ${NAME}:\t${HELPTEXT} [${CURRENT_VALUE}]"

    # find default value
    local DEFAULT_VALUE="${LINE#*=\"\${*:-}"
    DEFAULT_VALUE="${DEFAULT_VALUE%%\}\"*}"
    if [[ "${#DEFAULT_VALUE}" && "${CURRENT_VALUE}" != "${DEFAULT_VALUE}" ]] ; then
      echo ", default is '${DEFAULT_VALUE}'"
    else
      echo ""
    fi
  done < "${BASH_SOURCE}"
}

e.g.

BASE_DIR="${BASE_DIR:-$(pwd)}"         #@@ base directory for ssh tunnel
KEY_DIR="${KEY_DIR:-${BASE_DIR}/.ssh}" #@@ ssh keys for tunnel