Skip to content

Instantly share code, notes, and snippets.

@fortejas
Created July 9, 2019 08:37
Show Gist options
  • Save fortejas/bae02670bd8fb817dd23f146b79370d2 to your computer and use it in GitHub Desktop.
Save fortejas/bae02670bd8fb817dd23f146b79370d2 to your computer and use it in GitHub Desktop.
AWS Example - App Mesh on EC2 with CloudFormation
# Copyright 2019 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Parameters:
ImageId:
Type: AWS::SSM::Parameter::Value<String>
Description: Image to use for the client and server instances.
Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2
Namespace:
Type: String
Description: The namespace to create this mesh under.
Default: cluster.internal
KeyName:
Type: String
Description: The SSH KeyName to add to the two instances.
Default: aws-eu-central-1
InstanceType:
Type: String
Description: The instance type to use for the two instances.
Default: t3.medium
Resources:
# Create a Basic VPC.
VPC:
Type: AWS::EC2::VPC
Properties:
CidrBlock: 10.0.0.0/16
EnableDnsHostnames: true
EnableDnsSupport: true
# Add a Subnet.
Subnet:
Type: AWS::EC2::Subnet
Properties:
CidrBlock: 10.0.0.0/24
MapPublicIpOnLaunch: false
VpcId: !Ref VPC
# Our Subnet will be Public so we need an internet gateway.
IGW:
Type: AWS::EC2::InternetGateway
# Attach the IGW to our VPC.
IGWAttachment:
Type: AWS::EC2::VPCGatewayAttachment
Properties:
InternetGatewayId: !Ref IGW
VpcId: !Ref VPC
# Create a Route Table for our subnet.
RouteTable:
Type: AWS::EC2::RouteTable
Properties:
VpcId: !Ref VPC
# Link the route table to our subnet.
RouteTableAssociation:
Type: AWS::EC2::SubnetRouteTableAssociation
Properties:
RouteTableId: !Ref RouteTable
SubnetId: !Ref Subnet
# Add a route to the internet through our IGW.
InternetAccessRoute:
Type: AWS::EC2::Route
Properties:
DestinationCidrBlock: 0.0.0.0/0
GatewayId: !Ref IGW
RouteTableId: !Ref RouteTable
# Create an IAM Role for our two instances.
AppMeshIAMRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Principal:
Service:
- "ec2.amazonaws.com"
Action:
- "sts:AssumeRole"
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSAppMeshEnvoyAccess
- arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
# Create an IAM Profile for our instances.
EC2InstanceProfile:
Type: AWS::IAM::InstanceProfile
Properties:
Roles:
- !Ref AppMeshIAMRole
# Add a private service discovery namespace for our mesh.
ServiceDiscovery:
Type: AWS::ServiceDiscovery::PrivateDnsNamespace
Properties:
Description: Private DNS Namespace for AppMesh
Name: !Ref Namespace
Vpc: !Ref VPC
# Create a service discovery service for our server.
ServerServiceDiscovery:
Type: AWS::ServiceDiscovery::Service
Properties:
Description: Server for our mesh
Name: server
NamespaceId: !Ref ServiceDiscovery
HealthCheckCustomConfig:
FailureThreshold: 1
DnsConfig:
DnsRecords:
- TTL: 60
Type: A
NamespaceId: !Ref ServiceDiscovery
RoutingPolicy: WEIGHTED
# Create a service discovery service for our client.
ClientServiceDiscovery:
Type: AWS::ServiceDiscovery::Service
Properties:
Description: Client for our mesh
Name: client
NamespaceId: !Ref ServiceDiscovery
HealthCheckCustomConfig:
FailureThreshold: 1
DnsConfig:
DnsRecords:
- TTL: 60
Type: A
NamespaceId: !Ref ServiceDiscovery
RoutingPolicy: WEIGHTED
# Create our Mesh.
Mesh:
Type: AWS::AppMesh::Mesh
Properties:
MeshName: my-new-mesh
Spec:
EgressFilter:
Type: DROP_ALL
# Create a virtual node for our server.
ServerNode:
Type: AWS::AppMesh::VirtualNode
Properties:
MeshName: !GetAtt Mesh.MeshName
Spec:
Listeners:
- PortMapping:
Port: 80
Protocol: http
HealthCheck:
HealthyThreshold: 5
IntervalMillis: 30000
Path: /
Port: 80
Protocol: http
TimeoutMillis: 5000
UnhealthyThreshold: 2
Logging:
AccessLog:
File:
Path: /dev/stdout
ServiceDiscovery:
AWSCloudMap:
NamespaceName: !Ref Namespace
ServiceName: !GetAtt ServerServiceDiscovery.Name
VirtualNodeName: server
# Create a virtual service for our server node.
ServerService:
Type: AWS::AppMesh::VirtualService
Properties:
MeshName: !GetAtt Mesh.MeshName
VirtualServiceName: !Sub server.${Namespace}
Spec:
Provider:
VirtualNode:
VirtualNodeName: !GetAtt ServerNode.VirtualNodeName
# Create a virtual node for our client.
ClientNode:
Type: AWS::AppMesh::VirtualNode
Properties:
MeshName: !GetAtt Mesh.MeshName
Spec:
Backends:
- VirtualService:
VirtualServiceName: !GetAtt ServerService.VirtualServiceName
Logging:
AccessLog:
File:
Path: /dev/stdout
Listeners:
- PortMapping:
Port: 80
Protocol: http
ServiceDiscovery:
AWSCloudMap:
NamespaceName: !Ref Namespace
ServiceName: !GetAtt ClientServiceDiscovery.Name
VirtualNodeName: client
# Create a security group for our client and server instances.
SecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupDescription: Allow http to client host
VpcId: !Ref VPC
SecurityGroupIngress:
- CidrIp: 0.0.0.0/0
Description: Allow SSH Access to these Nodes
FromPort: 22
IpProtocol: tcp
ToPort: 22
# Allow all trafic between instances with this security group.
SecurityGroupIngress:
Type: AWS::EC2::SecurityGroupIngress
Properties:
GroupId: !Ref SecurityGroup
IpProtocol: -1
ToPort: -1
FromPort: -1
SourceSecurityGroupId: !Ref SecurityGroup
# Create a server instance.
EC2Server:
DependsOn:
- ServerNode
- RouteTableAssociation
Type: AWS::EC2::Instance
CreationPolicy:
ResourceSignal:
Timeout: PT5M
Metadata:
AWS::CloudFormation::Init:
configSets:
ConfigureAppMesh:
- install-httpd
- install-docker
- pull-envoy
- setup-proxy
- run-envoy
install-httpd:
packages:
yum:
httpd: []
services:
sysvinit:
httpd:
enabled: true
ensureRunning: true
commands:
add-page:
command: echo 'ok' > /var/www/html/index.html
install-docker:
commands:
install:
command: amazon-linux-extras install -y docker
services:
sysvinit:
docker:
enabled: true
ensureRunning: true
pull-envoy:
files:
/tmp/pull-envoy.sh:
mode: "000644"
owner: "root"
group: "root"
content: !Sub |
#!/usr/bin/env bash -ex
$(aws ecr get-login --no-include-email --region us-west-2 --registry-ids 111345817488)
docker pull 111345817488.dkr.ecr.us-west-2.amazonaws.com/aws-appmesh-envoy:v1.9.1.0-prod
commands:
envoy-start:
command: bash /tmp/pull-envoy.sh
run-envoy:
files:
/tmp/run-envoy.sh:
mode: "000644"
owner: "root"
group: "root"
content: !Sub |
#!/usr/bin/env bash -ex
systemctl restart docker
docker run --detach \
--env APPMESH_VIRTUAL_NODE_NAME=mesh/my-new-mesh/virtualNode/server \
-u 1337 \
--network host \
--restart always \
111345817488.dkr.ecr.us-west-2.amazonaws.com/aws-appmesh-envoy:v1.9.1.0-prod
commands:
start-envoy-container:
command: bash /tmp/run-envoy.sh
setup-proxy:
files:
/tmp/proxy-setup.sh:
mode: "000644"
owner: "root"
group: "root"
content: !Sub |
#!/bin/bash -e
#
# Start of configurable options
#
APPMESH_START_ENABLED="1"
APPMESH_IGNORE_UID="1337"
APPMESH_APP_PORTS="80"
APPMESH_ENVOY_EGRESS_PORT="15001"
APPMESH_ENVOY_INGRESS_PORT="15000"
APPMESH_EGRESS_IGNORED_IP="169.254.169.254,169.254.170.2"
# Enable routing on the application start.
[ -z "$APPMESH_START_ENABLED" ] && APPMESH_START_ENABLED="0"
# Egress traffic from the processess owned by the following UID/GID will be ignored.
if [ -z "$APPMESH_IGNORE_UID" ] && [ -z "$APPMESH_IGNORE_GID" ]; then
echo "Variables APPMESH_IGNORE_UID and/or APPMESH_IGNORE_GID must be set."
echo "Envoy must run under those IDs to be able to properly route it's egress traffic."
exit 1
fi
# Port numbers Application and Envoy are listening on.
if [ -z "$APPMESH_ENVOY_INGRESS_PORT" ] || [ -z "$APPMESH_ENVOY_EGRESS_PORT" ] || [ -z "$APPMESH_APP_PORTS" ]; then
echo "All of APPMESH_ENVOY_INGRESS_PORT, APPMESH_ENVOY_EGRESS_PORT and APPMESH_APP_PORTS variables must be set."
echo "If any one of them is not set we will not be able to route either ingress, egress, or both directions."
exit 1
fi
# Comma separated list of ports for which egress traffic will be ignored, we always refuse to route SSH traffic.
if [ -z "$APPMESH_EGRESS_IGNORED_PORTS" ]; then
APPMESH_EGRESS_IGNORED_PORTS="22"
else
APPMESH_EGRESS_IGNORED_PORTS="$APPMESH_EGRESS_IGNORED_PORTS,22"
fi
#
# End of configurable options
#
APPMESH_LOCAL_ROUTE_TABLE_ID="100"
APPMESH_PACKET_MARK="0x1e7700ce"
function initialize() {
echo "=== Initializing ==="
iptables -t mangle -N APPMESH_INGRESS
iptables -t nat -N APPMESH_INGRESS
iptables -t nat -N APPMESH_EGRESS
ip rule add fwmark "$APPMESH_PACKET_MARK" lookup $APPMESH_LOCAL_ROUTE_TABLE_ID
ip route add local default dev lo table $APPMESH_LOCAL_ROUTE_TABLE_ID
}
function enable_egress_routing() {
# Stuff to ignore
[ ! -z "$APPMESH_IGNORE_UID" ] && \
iptables -t nat -A APPMESH_EGRESS \
-m owner --uid-owner $APPMESH_IGNORE_UID \
-j RETURN
[ ! -z "$APPMESH_IGNORE_GID" ] && \
iptables -t nat -A APPMESH_EGRESS \
-m owner --gid-owner $APPMESH_IGNORE_GID \
-j RETURN
[ ! -z "$APPMESH_EGRESS_IGNORED_PORTS" ] && \
iptables -t nat -A APPMESH_EGRESS \
-p tcp \
-m multiport --dports "$APPMESH_EGRESS_IGNORED_PORTS" \
-j RETURN
[ ! -z "$APPMESH_EGRESS_IGNORED_IP" ] && \
iptables -t nat -A APPMESH_EGRESS \
-p tcp \
-d "$APPMESH_EGRESS_IGNORED_IP" \
-j RETURN
# Redirect everything that is not ignored
iptables -t nat -A APPMESH_EGRESS \
-p tcp \
-j REDIRECT --to $APPMESH_ENVOY_EGRESS_PORT
# Apply APPMESH_EGRESS chain to non local traffic
iptables -t nat -A OUTPUT \
-p tcp \
-m addrtype ! --dst-type LOCAL \
-j APPMESH_EGRESS
}
function enable_ingress_redirect_routing() {
# Route everything arriving at the application port to Envoy
iptables -t nat -A APPMESH_INGRESS \
-p tcp \
-m multiport --dports "$APPMESH_APP_PORTS" \
-j REDIRECT --to-port "$APPMESH_ENVOY_INGRESS_PORT"
# Apply AppMesh ingress chain to everything non-local
iptables -t nat -A PREROUTING \
-p tcp \
-m addrtype ! --src-type LOCAL \
-j APPMESH_INGRESS
}
function enable_routing() {
echo "=== Enabling routing ==="
enable_egress_routing
enable_ingress_redirect_routing
}
function disable_routing() {
echo "=== Disabling routing ==="
iptables -F
iptables -F -t nat
iptables -F -t mangle
}
function dump_status() {
echo "=== Routing rules ==="
ip rule
echo "=== AppMesh routing table ==="
ip route list table $APPMESH_LOCAL_ROUTE_TABLE_ID
echo "=== iptables FORWARD table ==="
iptables -L -v -n
echo "=== iptables NAT table ==="
iptables -t nat -L -v -n
echo "=== iptables MANGLE table ==="
iptables -t mangle -L -v -n
}
function main_loop() {
echo "=== Entering main loop ==="
while read -p '> ' cmd; do
case "$cmd" in
"quit")
break
;;
"status")
dump_status
;;
"enable")
enable_routing
;;
"disable")
disable_routing
;;
*)
echo "Available commands: quit, status, enable, disable"
;;
esac
done
}
function print_config() {
echo "=== Input configuration ==="
env | grep APPMESH_ || true
}
print_config
initialize
if [ "$APPMESH_START_ENABLED" == "1" ]; then
enable_routing
fi
main_loop
commands:
setup-proxy:
command: bash /tmp/proxy-setup.sh enable
Properties:
ImageId: !Ref ImageId
KeyName: !Ref KeyName
InstanceType: !Ref InstanceType
IamInstanceProfile: !Ref EC2InstanceProfile
NetworkInterfaces:
- AssociatePublicIpAddress: "true"
DeviceIndex: "0"
GroupSet:
- Ref: SecurityGroup
SubnetId: !Ref Subnet
Tags:
- Key: Name
Value: EC2Server
UserData: !Base64
Fn::Join:
- ''
- - |
#!/bin/bash -x
- |
# Install the files and packages from the metadata
- '/opt/aws/bin/cfn-init -v '
- ' --stack '
- !Ref 'AWS::StackName'
- ' --resource EC2Server '
- ' --configsets ConfigureAppMesh '
- ' --region '
- !Ref 'AWS::Region'
- |+
- |
# Signal the status from cfn-init
- '/opt/aws/bin/cfn-signal -e $? '
- ' --stack '
- !Ref 'AWS::StackName'
- ' --resource EC2Server '
- ' --region '
- !Ref 'AWS::Region'
- |+
# Add the server into service discovery.
ServerServiceDiscoveryInstance:
Type: AWS::ServiceDiscovery::Instance
Properties:
InstanceAttributes:
AWS_INSTANCE_IPV4: !GetAtt EC2Server.PrivateIp
ServiceId: !Ref ServerServiceDiscovery
# Create the client instance.
EC2Client:
DependsOn:
- ClientNode
- RouteTableAssociation
Type: AWS::EC2::Instance
CreationPolicy:
ResourceSignal:
Timeout: PT5M
Metadata:
AWS::CloudFormation::Init:
configSets:
ConfigureAppMesh:
- install-httpd
- install-docker
- pull-envoy
- setup-proxy
- run-envoy
install-httpd:
packages:
yum:
httpd: []
services:
sysvinit:
httpd:
enabled: true
ensureRunning: true
install-docker:
commands:
install:
command: amazon-linux-extras install -y docker
services:
sysvinit:
docker:
enabled: true
ensureRunning: true
pull-envoy:
files:
/tmp/pull-envoy.sh:
mode: "000644"
owner: "root"
group: "root"
content: !Sub |
#!/usr/bin/env bash -ex
$(aws ecr get-login --no-include-email --region us-west-2 --registry-ids 111345817488)
docker pull 111345817488.dkr.ecr.us-west-2.amazonaws.com/aws-appmesh-envoy:v1.9.1.0-prod
commands:
envoy-start:
command: bash /tmp/pull-envoy.sh
run-envoy:
files:
/tmp/run-envoy.sh:
mode: "000644"
owner: "root"
group: "root"
content: !Sub |
#!/usr/bin/env bash -ex
systemctl restart docker
docker run --detach \
--env APPMESH_VIRTUAL_NODE_NAME=mesh/my-new-mesh/virtualNode/client \
-u 1337 \
--network host \
--restart always \
111345817488.dkr.ecr.us-west-2.amazonaws.com/aws-appmesh-envoy:v1.9.1.0-prod
commands:
start-envoy-container:
command: bash /tmp/run-envoy.sh
setup-proxy:
files:
/tmp/proxy-setup.sh:
mode: "000644"
owner: "root"
group: "root"
content: !Sub |
#!/bin/bash -e
#
# Start of configurable options
#
APPMESH_START_ENABLED="1"
APPMESH_IGNORE_UID="1337"
APPMESH_APP_PORTS="80"
APPMESH_ENVOY_EGRESS_PORT="15001"
APPMESH_ENVOY_INGRESS_PORT="15000"
APPMESH_EGRESS_IGNORED_IP="169.254.169.254,169.254.170.2"
# Enable routing on the application start.
[ -z "$APPMESH_START_ENABLED" ] && APPMESH_START_ENABLED="0"
# Egress traffic from the processess owned by the following UID/GID will be ignored.
if [ -z "$APPMESH_IGNORE_UID" ] && [ -z "$APPMESH_IGNORE_GID" ]; then
echo "Variables APPMESH_IGNORE_UID and/or APPMESH_IGNORE_GID must be set."
echo "Envoy must run under those IDs to be able to properly route it's egress traffic."
exit 1
fi
# Port numbers Application and Envoy are listening on.
if [ -z "$APPMESH_ENVOY_INGRESS_PORT" ] || [ -z "$APPMESH_ENVOY_EGRESS_PORT" ] || [ -z "$APPMESH_APP_PORTS" ]; then
echo "All of APPMESH_ENVOY_INGRESS_PORT, APPMESH_ENVOY_EGRESS_PORT and APPMESH_APP_PORTS variables must be set."
echo "If any one of them is not set we will not be able to route either ingress, egress, or both directions."
exit 1
fi
# Comma separated list of ports for which egress traffic will be ignored, we always refuse to route SSH traffic.
if [ -z "$APPMESH_EGRESS_IGNORED_PORTS" ]; then
APPMESH_EGRESS_IGNORED_PORTS="22"
else
APPMESH_EGRESS_IGNORED_PORTS="$APPMESH_EGRESS_IGNORED_PORTS,22"
fi
#
# End of configurable options
#
APPMESH_LOCAL_ROUTE_TABLE_ID="100"
APPMESH_PACKET_MARK="0x1e7700ce"
function initialize() {
echo "=== Initializing ==="
iptables -t mangle -N APPMESH_INGRESS
iptables -t nat -N APPMESH_INGRESS
iptables -t nat -N APPMESH_EGRESS
ip rule add fwmark "$APPMESH_PACKET_MARK" lookup $APPMESH_LOCAL_ROUTE_TABLE_ID
ip route add local default dev lo table $APPMESH_LOCAL_ROUTE_TABLE_ID
}
function enable_egress_routing() {
# Stuff to ignore
[ ! -z "$APPMESH_IGNORE_UID" ] && \
iptables -t nat -A APPMESH_EGRESS \
-m owner --uid-owner $APPMESH_IGNORE_UID \
-j RETURN
[ ! -z "$APPMESH_IGNORE_GID" ] && \
iptables -t nat -A APPMESH_EGRESS \
-m owner --gid-owner $APPMESH_IGNORE_GID \
-j RETURN
[ ! -z "$APPMESH_EGRESS_IGNORED_PORTS" ] && \
iptables -t nat -A APPMESH_EGRESS \
-p tcp \
-m multiport --dports "$APPMESH_EGRESS_IGNORED_PORTS" \
-j RETURN
[ ! -z "$APPMESH_EGRESS_IGNORED_IP" ] && \
iptables -t nat -A APPMESH_EGRESS \
-p tcp \
-d "$APPMESH_EGRESS_IGNORED_IP" \
-j RETURN
# Redirect everything that is not ignored
iptables -t nat -A APPMESH_EGRESS \
-p tcp \
-j REDIRECT --to $APPMESH_ENVOY_EGRESS_PORT
# Apply APPMESH_EGRESS chain to non local traffic
iptables -t nat -A OUTPUT \
-p tcp \
-m addrtype ! --dst-type LOCAL \
-j APPMESH_EGRESS
}
function enable_ingress_redirect_routing() {
# Route everything arriving at the application port to Envoy
iptables -t nat -A APPMESH_INGRESS \
-p tcp \
-m multiport --dports "$APPMESH_APP_PORTS" \
-j REDIRECT --to-port "$APPMESH_ENVOY_INGRESS_PORT"
# Apply AppMesh ingress chain to everything non-local
iptables -t nat -A PREROUTING \
-p tcp \
-m addrtype ! --src-type LOCAL \
-j APPMESH_INGRESS
}
function enable_routing() {
echo "=== Enabling routing ==="
enable_egress_routing
enable_ingress_redirect_routing
}
function disable_routing() {
echo "=== Disabling routing ==="
iptables -F
iptables -F -t nat
iptables -F -t mangle
}
function dump_status() {
echo "=== Routing rules ==="
ip rule
echo "=== AppMesh routing table ==="
ip route list table $APPMESH_LOCAL_ROUTE_TABLE_ID
echo "=== iptables FORWARD table ==="
iptables -L -v -n
echo "=== iptables NAT table ==="
iptables -t nat -L -v -n
echo "=== iptables MANGLE table ==="
iptables -t mangle -L -v -n
}
function main_loop() {
echo "=== Entering main loop ==="
while read -p '> ' cmd; do
case "$cmd" in
"quit")
break
;;
"status")
dump_status
;;
"enable")
enable_routing
;;
"disable")
disable_routing
;;
*)
echo "Available commands: quit, status, enable, disable"
;;
esac
done
}
function print_config() {
echo "=== Input configuration ==="
env | grep APPMESH_ || true
}
print_config
initialize
if [ "$APPMESH_START_ENABLED" == "1" ]; then
enable_routing
fi
main_loop
commands:
setup-proxy:
command: bash /tmp/proxy-setup.sh enable
Properties:
ImageId: !Ref ImageId
KeyName: !Ref KeyName
InstanceType: !Ref InstanceType
IamInstanceProfile: !Ref EC2InstanceProfile
NetworkInterfaces:
- AssociatePublicIpAddress: true
DeviceIndex: "0"
GroupSet:
- Ref: SecurityGroup
SubnetId: !Ref Subnet
Tags:
- Key: Name
Value: EC2Client
UserData: !Base64
Fn::Join:
- ''
- - |
#!/bin/bash -x
- |
# Install the files and packages from the metadata
- '/opt/aws/bin/cfn-init -v '
- ' --stack '
- !Ref 'AWS::StackName'
- ' --resource EC2Client '
- ' --configsets ConfigureAppMesh '
- ' --region '
- !Ref 'AWS::Region'
- |+
- |
# Signal the status from cfn-init
- '/opt/aws/bin/cfn-signal -e $? '
- ' --stack '
- !Ref 'AWS::StackName'
- ' --resource EC2Client '
- ' --region '
- !Ref 'AWS::Region'
- |+
# Add the client into service discovery.
ClientServiceDiscoveryInstance:
Type: AWS::ServiceDiscovery::Instance
Properties:
InstanceAttributes:
AWS_INSTANCE_IPV4: !GetAtt EC2Client.PrivateIp
ServiceId: !Ref ClientServiceDiscovery
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment