EC2 Userdata with Prometheus Node Exporter and Loki for ECS

userdata_ecs.txt

#!/bin/bash
AWS_ACCOUNT="dev"
CLUSTER_NAME="aws-qa-ecs"
ENVIRONMENT_NAME="qa"
NODE_EXPORTER_VERSION="1.0.1"
NODE_EXPORTER_USER="node_exporter"
MY_HOSTNAME="$(curl -s http://169.254.169.254/latest/meta-data/local-hostname)"
INSTANCE_ID="$(curl -s http://instance-data/latest/meta-data/instance-id)"
INSTANCE_LIFECYCLE="$(curl -s http://169.254.169.254/latest/meta-data/instance-life-cycle)"
REGION="$(curl -s http://instance-data/latest/meta-data/placement/availability-zone | rev | cut -c 2- | rev)"

echo "ECS_CLUSTER=${CLUSTER_NAME}" >> /etc/ecs/ecs.config
echo "ECS_AVAILABLE_LOGGING_DRIVERS=[\"json-file\",\"awslogs\"]" >> /etc/ecs/ecs.config
echo "ECS_INSTANCE_ATTRIBUTES={\"environment\":\"${ENVIRONMENT_NAME}\"}" >> /etc/ecs/ecs.config

docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions
sudo yum install wget -y

cat > /etc/docker/daemon.json << EOF
{
    "debug" : true,
    "log-driver": "loki",
    "log-opts": {
        "loki-url": "https://x:x@loki.domain.local/loki/api/v1/push",
        "loki-batch-size": "300",
        "loki-external-labels": "job=qa/dockerlogs,container_name={{.Name}},cluster_name=${CLUSTER_NAME},instanceid=${INSTANCE_ID},instance_lifecycle=${INSTANCE_LIFECYCLE},aws_account=${AWS_ACCOUNT},environment=${ENVIRONMENT_NAME}"
    }
}
EOF

sudo systemctl restart docker

id -u ${NODE_EXPORTER_USER} &> /dev/null && EXIT_CODE=${?} || EXIT_CODE=${?}
if [ ${EXIT_CODE} == 1 ]
  then
    useradd --no-create-home --shell /bin/false ${NODE_EXPORTER_USER}
fi

wget https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXPORTER_VERSION}/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz
tar -xf node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz
cp node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64/node_exporter /usr/bin/
chown ${NODE_EXPORTER_USER}:${NODE_EXPORTER_USER} /usr/bin/node_exporter
rm -rf node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64*

cat > /etc/systemd/system/node_exporter.service << EOF
[Unit]
Description=Node Exporter
Wants=network-online.target
After=network-online.target
StartLimitIntervalSec=500
StartLimitBurst=5
[Service]
User=${NODE_EXPORTER_USER}
Group=${NODE_EXPORTER_USER}
Type=simple
Restart=on-failure
RestartSec=5s
ExecStart=/usr/bin/node_exporter
[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl start node_exporter