script to download atlas logs. requires "project owner" access

download_atlas_logs.sh

#!/bin/sh

set -eo pipefail
# Group ID is synonymous with projectId (in the ui)
GROUP_ID=""

# Configuration -- https://cloud.mongodb.com/v2/${GROUP_ID}/#access/apiKeys
# Note: the API key requires "Project owner" access in order to download the logs

if [ -z "$ATLAS_PUBLIC_KEY" ] || [ -z "$ATLAS_API_KEY" ]; then
  echo "You need to run this with ATLAS_PUBLIC_KEY and ATLAS_API_KEY in the env"
  exit 1
fi

LAST_FETCHED=$(consul kv get "atlas-logs/$GROUP_ID/last_fetched")
NOW=$(date +%s)

HOSTNAMES=$(curl --user "$ATLAS_PUBLIC_KEY:$ATLAS_API_KEY" --digest --silent --show-error --fail  --request GET "https://cloud.mongodb.com/api/atlas/v1.0/groups/$GROUP_ID/processes" | jq -r '.results[] | .hostname')

for hostName in $HOSTNAMES; do
  FILENAME="${hostName}-${LAST_FETCHED}-${NOW}-mongodb.log.gz"
  # https://docs.atlas.mongodb.com/reference/api/logs/
  curl --user "$ATLAS_PUBLIC_KEY:$ATLAS_API_KEY" --digest \
    --silent \
    --show-error --fail\
    --header 'Accept: application/gzip' \
    --request GET "https://cloud.mongodb.com/api/atlas/v1.0/groups/${GROUP_ID}/clusters/${hostName}/logs/mongodb.gz?startDate=${LAST_FETCHED}&endDate=${NOW}" \
    --output "$FILENAME"
  if [ -s "$FILENAME" ]; then zcat "$FILENAME"; fi
  rm "$FILENAME"
done
consul kv put "atlas-logs/$GROUP_ID/last_fetched" "$NOW"

sleep 30