Shell script to collect Nomad stderr logs for each job. Recommended to run through cron on a 5-10 minute interval.

nomad_logs.sh

# bash script to collect Nomad stderr (aka outer build container) logs
# run via cron on 5-10 minute interval, please ensure you turn off this service after usage

#! /bin/bash

set -eu

output_dir="nomad-stderr-logs"
nomad_log_dir_max_bytes=1073741824

generate_logs() {
  if [ $(docker exec -it nomad nomad status | wc -l) -gt 1 ]; then
    job_ids=($(docker exec -it nomad nomad status | awk '{if (NR!=1)print $1}'))
    for i in "${!job_ids[@]}"; do
      if [ ! -f "$output_dir/${job_ids[i]}.log" ]; then
        docker exec -it nomad nomad status ${job_ids[i]} | sed '$!d' | awk '{print $1}' | xargs docker exec -it nomad nomad logs -stderr > $output_dir/${job_ids[i]}.log
        echo "${job_ids[i]} log successfully created."
      fi
    done
  else
    echo "$(date +%Y%m%d%H%M%S) No job logs to collect."
    exit 0
  fi
}

check_dir_size() {
  if [ $(du -b $output_dir | awk '{print $1}') -gt $nomad_log_dir_max_bytes ]; then
    echo "$(date +%Y%m%d%H%M%S) ERROR Nomad log directory has exceeded $nomad_log_dir_max_bytes bytes. Please stop the Nomad log collector script, or change the max size value from within the script"
    exit 1
  fi
}

mkdir -p $output_dir
check_dir_size
generate_logs
echo "$(date +%Y%m%d%H%M%S) Directory $output_dir updated with most recent jobs"