Download full AWS CloudWatch log stream

download_log_stream.sh

#!/bin/bash -e

log_group_name="/aws/sagemaker/TrainingJobs"
log_stream_name=${1}
output_file=$(echo "${1}.txt" | sed 's/\//_/g')
next_token=""

echo "Storing full logs to: $output_file"

# Clear output file
>"$output_file"

# Loop to handle pagination
page=1
last_token=""
while true; do
    if [ -z "$next_token" ]; then
        response=$(aws logs get-log-events --log-group-name "$log_group_name" --log-stream-name "$log_stream_name" --start-from-head)
    else
        response=$(aws logs get-log-events --log-group-name "$log_group_name" --log-stream-name "$log_stream_name" --next-token "$next_token")
    fi
    echo "$response" | jq -r '.events[] | @text "\(.timestamp) \(.message)"' >>"$output_file"
    next_token=$(echo "$response" | jq -r '.nextForwardToken')
    if [ "$next_token" == "null" ] || [ "$next_token" == "$last_token" ]; then
        break
    fi
    ((page++))
    echo "Fetching page ${page} (next token: $next_token)"
    last_token="$next_token"
done