15-self-adaptation-infra.sh

export AWS_ACCESS_KEY_ID=[...]

export AWS_SECRET_ACCESS_KEY=[...]

export AWS_DEFAULT_REGION=us-east-1

export STACK_NAME=devops22

export KEY_NAME=devops22

aws cloudformation create-stack \
    --template-url https://editions-us-east-1.s3.amazonaws.com/aws/stable/Docker.tmpl \
    --capabilities CAPABILITY_IAM \
    --stack-name $STACK_NAME \
    --parameters \
    ParameterKey=ManagerSize,ParameterValue=3 \
    ParameterKey=ClusterSize,ParameterValue=0 \
    ParameterKey=KeyName,ParameterValue=$KEY_NAME \
    ParameterKey=EnableSystemPrune,ParameterValue=yes \
    ParameterKey=EnableCloudWatchLogs,ParameterValue=no \
    ParameterKey=EnableCloudStorEfs,ParameterValue=yes \
    ParameterKey=ManagerInstanceType,ParameterValue=t2.small \
    ParameterKey=InstanceType,ParameterValue=t2.small

aws cloudformation describe-stacks \
    --stack-name $STACK_NAME | \
    jq -r ".Stacks[0].StackStatus"

CLUSTER_DNS=$(aws cloudformation \
    describe-stacks \
    --stack-name $STACK_NAME | \
    jq -r ".Stacks[0].Outputs[] | \
    select(.OutputKey==\"DefaultDNSTarget\")\
    .OutputValue")

CLUSTER_IP=$(aws ec2 describe-instances \
    | jq -r ".Reservations[] \
    .Instances[] \
    | select(.SecurityGroups[].GroupName \
    | contains(\"$STACK_NAME-ManagerVpcSG\"))\
    .PublicIpAddress" \
    | tail -n 1)

echo "
export CLUSTER_DNS=$CLUSTER_DNS
export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
export AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION
"

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

echo "
export CLUSTER_DNS=[...]
export AWS_ACCESS_KEY_ID=[...]
export AWS_SECRET_ACCESS_KEY=[...]
export AWS_DEFAULT_REGION=[...]
">creds

curl -o aws-services-15.sh \
    https://raw.githubusercontent.com/vfarcic/docker-flow-monitor/master/scripts/aws-services-15.sh

chmod +x aws-services-15.sh

source creds

./aws-services-15.sh

docker stack ls

exit

aws autoscaling \
    describe-auto-scaling-groups \
    | jq "."

aws autoscaling \
    describe-auto-scaling-groups \
    | jq -r ".AutoScalingGroups[] \
    | select(.AutoScalingGroupName \
    | startswith(\"$STACK_NAME-NodeAsg-\"))\
    .AutoScalingGroupName"

aws autoscaling \
    describe-auto-scaling-groups \
    | jq -r ".AutoScalingGroups[] \
    | select(.AutoScalingGroupName \
    | startswith(\"$STACK_NAME-NodeAsg-\"))\
    .DesiredCapacity"

ASG_NAME=$(aws autoscaling \
    describe-auto-scaling-groups \
    | jq -r ".AutoScalingGroups[] \
    | select(.AutoScalingGroupName \
    | startswith(\"$STACK_NAME-NodeAsg-\"))\
    .AutoScalingGroupName")

aws autoscaling \
    update-auto-scaling-group \
    --auto-scaling-group-name $ASG_NAME \
    --desired-capacity 1

aws autoscaling \
    describe-auto-scaling-groups \
    --auto-scaling-group-names $ASG_NAME \
    | jq ".AutoScalingGroups[0]\
    .DesiredCapacity"

aws ec2 describe-instances | jq -r \
    ".Reservations[].Instances[] \
    | select(.SecurityGroups[].GroupName \
    | startswith(\"$STACK_NAME-NodeVpcSG\"))\
    .InstanceId"

INSTANCE_ID=$(aws ec2 \
    describe-instances | jq -r \
    ".Reservations[].Instances[] \
    | select(.SecurityGroups[].GroupName \
    | startswith(\"$STACK_NAME-NodeVpcSG\"))\
    .InstanceId")

aws ec2 describe-instance-status \
    --instance-ids $INSTANCE_ID \
    | jq -r ".InstanceStatuses[0]\
    .InstanceStatus.Status"

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

docker node ls

exit

aws autoscaling \
    update-auto-scaling-group \
    --auto-scaling-group-name $ASG_NAME \
    --desired-capacity 0

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

docker node ls

curl "https://raw.githubusercontent.com/vfarcic/docker-aws-cli/master/Dockerfile"

source creds

docker container run --rm \
    -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
    -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
    -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION \
    vfarcic/aws-cli \
    aws ec2 describe-instances

curl "https://raw.githubusercontent.com/vfarcic/docker-aws-cli/master/docker-compose.yml"

curl -o jenkins.yml \
    https://raw.githubusercontent.com/vfarcic/docker-flow-monitor/master/stacks/jenkins-aws-secret.yml

cat jenkins.yml

source creds

echo "
export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
export AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION
export STACK_NAME=devops22
" | docker secret create aws -

docker stack deploy \
    -c jenkins.yml jenkins

exit

open "http://$CLUSTER_DNS/jenkins/configure"

open "http://$CLUSTER_DNS/jenkins/view/all/newJob"

open "http://$CLUSTER_DNS/jenkins/blue/organizations/jenkins/aws-scale/activity"

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

docker node ls

exit

curl -XPOST -i \
    "http://$CLUSTER_DNS/jenkins/job/aws-scale/buildWithParameters?token=DevOps22&scale=2"

open "http://$CLUSTER_DNS/jenkins/blue/organizations/jenkins/aws-scale/activity"

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

docker node ls

exit

curl -XPOST -i \
    "http://$CLUSTER_DNS/jenkins/job/aws-scale/buildWithParameters?token=DevOps22&scale=1"

open "http://$CLUSTER_DNS/jenkins/blue/organizations/jenkins/aws-scale/activity"

curl -XPOST -i \
    "http://$CLUSTER_DNS/jenkins/job/aws-scale/buildWithParameters?token=DevOps22&scale=-2"

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

docker node ls

exit

curl -XPOST -i \
    "http://$CLUSTER_DNS/jenkins/job/aws-scale/buildWithParameters?token=DevOps22&scale=-1"

open "http://$CLUSTER_DNS/monitor"

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

curl -o exporters.yml \
    https://raw.githubusercontent.com/vfarcic/docker-flow-monitor/master/stacks/exporters-aws.yml

cat exporters.yml

docker stack rm exporter

docker stack deploy -c exporters.yml \
    exporter

exit

open "http://$CLUSTER_DNS/monitor/alerts"

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

docker stack rm monitor

docker secret rm alert_manager_config

source creds

echo "route:
  group_by: [service,scale,type]
  repeat_interval: 30m
  group_interval: 30m
  receiver: 'slack'
  routes:
  - match:
      type: 'node'
      scale: 'up'
    receiver: 'jenkins-node-up'
  - match:
      type: 'node'
      scale: 'down'
    receiver: 'jenkins-node-down'
  - match:
      service: 'go-demo_main'
      scale: 'up'
    receiver: 'jenkins-go-demo_main-up'
  - match:
      service: 'go-demo_main'
      scale: 'down'
    receiver: 'jenkins-go-demo_main-down'

receivers:
  - name: 'slack'
    slack_configs:
      - send_resolved: true
        title: '[{{ .Status | toUpper }}] {{ .GroupLabels.service }} service is in danger!'
        title_link: 'http://$CLUSTER_DNS/monitor/alerts'
        text: '{{ .CommonAnnotations.summary}}'
        api_url: 'https://hooks.slack.com/services/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'
  - name: 'jenkins-go-demo_main-up'
    webhook_configs:
      - send_resolved: false
        url: 'http://$CLUSTER_DNS/jenkins/job/service-scale/buildWithParameters?token=DevOps22&service=go-demo_main&scale=1'
  - name: 'jenkins-go-demo_main-down'
    webhook_configs:
      - send_resolved: false
        url: 'http://$CLUSTER_DNS/jenkins/job/service-scale/buildWithParameters?token=DevOps22&service=go-demo_main&scale=-1'
  - name: 'jenkins-node-up'
    webhook_configs:
      - send_resolved: false
        url: 'http://$CLUSTER_DNS/jenkins/job/aws-scale/buildWithParameters?token=DevOps22&scale=1'
  - name: 'jenkins-node-down'
    webhook_configs:
      - send_resolved: false
        url: 'http://$CLUSTER_DNS/jenkins/job/aws-scale/buildWithParameters?token=DevOps22&scale=-1'
" | docker secret create alert_manager_config -

DOMAIN=$CLUSTER_DNS docker stack \
    deploy -c monitor.yml monitor

docker stack ps \
    -f desired-state=running monitor

docker service update \
    --label-add "com.df.alertIf.2=@node_mem_limit_total_above:0.1" \
    exporter_node-exporter-manager

exit

open "http://$CLUSTER_DNS/monitor/alerts"

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

docker service update \
    --label-add "com.df.alertIf.2=@node_mem_limit_total_above:0.8" \
    exporter_node-exporter-manager

docker service update \
    --label-add "com.df.alertIf.2=@node_mem_limit_total_above:0.1" \
    exporter_node-exporter-worker

exit

open "http://$CLUSTER_DNS/monitor/alerts"

open "http://$CLUSTER_DNS/jenkins/blue/organizations/jenkins/aws-scale/activity"

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

docker node ls

docker service update \
    --label-add "com.df.alertIf.2=@node_mem_limit_total_above:0.8" \
    exporter_node-exporter-worker

docker service update \
    --label-add "com.df.alertIf.3=@node_mem_limit_total_below:0.9" \
    exporter_node-exporter-worker

exit

open "http://$CLUSTER_DNS/monitor/alerts"

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

docker node ls

docker service update \
    --label-add "com.df.alertIf.3=@node_mem_limit_total_below:0.05" \
    exporter_node-exporter-worker

exit

docker service ls -q \
    -f label=com.df.reschedule=true

docker service update --env-add 'RESCHEDULE_DATE=${date}' ${service}

open "http://$CLUSTER_DNS/jenkins/job/aws-scale/configure"

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

curl -o go-demo.yml \
    https://raw.githubusercontent.com/vfarcic/docker-flow-monitor/master/stacks/go-demo-aws.yml

cat go-demo.yml

docker stack deploy -c go-demo.yml \
    go-demo

exit

curl -XPOST -i \
    "http://$CLUSTER_DNS/jenkins/job/aws-scale/buildWithParameters?token=DevOps22&scale=1"

open "http://$CLUSTER_DNS/jenkins/blue/organizations/jenkins/aws-scale/activity"

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

docker node ls

NODE_ID=[...]

docker node ps \
    -f desired-state=running $NODE_ID

docker node ps \
    -f desired-state=running $NODE_ID

exit

curl -XPOST -i \
    "http://$CLUSTER_DNS/jenkins/job/aws-scale/buildWithParameters?token=DevOps22&scale=-1"

open "http://$CLUSTER_DNS/monitor"

count(container_memory_usage_bytes{container_label_com_docker_swarm_service_name="go-demo_main"})

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

cat go-demo.yml

exit

open "http://$CLUSTER_DNS/monitor/alerts"

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

docker service scale \
    monitor_alert-manager=0

docker service scale \
    monitor_alert-manager=1

docker service update \
    --reserve-memory 1500M \
    --replicas 4 \
    go-demo_main

docker stack ps \
    -f desired-state=running go-demo

exit

open "http://$CLUSTER_DNS/monitor/alerts"

open "http://$CLUSTER_DNS/jenkins/blue/organizations/jenkins/aws-scale/activity"

ssh -i $KEY_NAME.pem docker@$CLUSTER_IP

docker node ls

docker stack ps \
    -f desired-state=running go-demo

exit

aws cloudformation delete-stack \
    --stack-name devops22