Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Example coredump uploader
#!/bin/bash
# Depends on zstd (>1.3.6) and aws-cli
set -o errexit
set -o nounset
set -o pipefail
# Set kernel.core_pattern = | coredump_uploader.sh %P %s %E
PID=$1
shift
SIGNAL=$1
shift
EXECUTABLE=$1
shift
# Set these to appropriate values for your environment
readonly BUCKET=
readonly APP_NAME=
readonly INSTANCE_ID=
readonly COREDUMP_BACKOFF_MINS=60
TARGET_STDERR="/proc/${PID}/fd/2"
if ! exec 2>>"${TARGET_STDERR}" >&2; then
# Redirect output into the target's stderr, if possible
exec 2> >(exec 2>&-; logger -t "coredump-uploader-${PID}")
exec >&2
fi
echo "${0##*/} initiated for PID ${PID}"
mkdir -p /run/coredumps
find /run/coredumps -mmin "+${COREDUMP_BACKOFF_MINS}" -type f -delete
LAST=$(find /run/coredumps -type f -printf "%t\n" | head -n1)
if [ -n "${LAST}" ]; then
# Back off without creating a file
echo "Last core dump was at ${LAST}; too soon to upload another."
exit
fi
mktemp -p /run/coredumps dump.XXXXXXXXXX
MACHINE_DATE=$(date "+%s.%N")
HUMAN_DATE=$(date -d "@${MACHINE_DATE}" '+%Y-%m-%dT%H:%M:%S.%N')
TWO_63=9223372036854775808
REVERSE_DATE=$(($TWO_63 - ${MACHINE_DATE/.})) # overflows in 2230
S3_KEY="s3://${BUCKET}/apps/${APP_NAME}/coredumps/${INSTANCE_ID}/${REVERSE_DATE}_${HUMAN_DATE}"
echo "Uploading to: ${S3_KEY}"
echo "Uploading metadata"
EXE_HASH=$(sha256sum /proc/$PID/exe | cut -d' ' -f1)
(
cat <<EOF
Date: $HUMAN_DATE
PID: $PID
Signal: $SIGNAL
Command: ${EXECUTABLE//!//}
SHA256(Executable): $EXE_HASH
EOF
lsof -p $PID -Pn
) | aws s3 cp - "${S3_KEY}/meta"
echo "Uploading tarball"
#XXX: which of these files duplicate data already present in the core dump?
#XXX: also, if any of these files are useful, should we get the same from each
# running thread
tar -C /proc/$PID -c \
cgroup \
cmdline \
cpuset \
comm \
environ \
gid_map \
io \
maps \
mountinfo \
sched \
schedstat \
smaps \
stack \
stat \
status \
uid_map \
wchan \
| aws s3 cp - "${S3_KEY}/proc.tar"
echo "Uploading core file"
echo "You may see 'Aborted (Core Dumped)' printed to the terminal before completion"
# If you can't use zstd, consider lz4 instead: it has compression throughput
# suitable for streaming uploads.
zstd --adapt | aws s3 cp - "${S3_KEY}/core.zst"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment