Download a CloudWatch Logs group

download.sh

#!/bin/bash

log_group=$1
list_command="aws logs describe-log-streams --log-group-name $log_group"
group_next_token=''

while true; do
  if [[ -n $group_next_token ]]; then
    response=$($list_command --next-token $group_next_token)
  else
    response=$($list_command)
  fi
  group_next_token=$(echo $response | jq -r '.NextToken')
  echo $response | jq -r '.logStreams[].logStreamName' | xargs -n 1 -P 20 ~/bin/get-stream.sh $log_group
  if [[ ${group_next_token} == "null" ]]; then
    break
  fi
done

get-stream.sh

#!/bin/bash

log_group=$1
stream=$2
stream_file=`echo $stream | sed 's%/%_%g'`
get_command="aws logs get-log-events --log-group-name $log_group --start-from-head"
stream_next_token=''
counter=0

mkdir -p streams/$stream_file

while true; do
  output_path=$(printf "streams/%s/%s-%05d.log.gz" $stream_file $stream_file $counter)
  echo "$output_path" 1>&2
  if [[ -n $stream_next_token ]]; then
    response=$($get_command --log-stream-name $stream --next-token $stream_next_token)
  else
    response=$($get_command --log-stream-name $stream)
  fi
  echo $response | jq -r '.events[].message' | gzip > $output_path
  stream_next_token=$(echo $response | jq -r '.nextForwardToken')
  if [[ ${#stream_next_token} -lt 40 ]]; then
    break
  fi
  counter=$((counter + 1))
done