HipChat + Hubot + CloudFormationで環境構築の自動化

aws_cloudformation.coffee

# Dependencies:
#   "aws2js": "0.6.12"
#   "underscore": "1.3.3"
#
# Configuration:
#   HUBOT_AWS_ACCESS_KEY_ID
#   HUBOT_AWS_SECRET_ACCESS_KEY
#   HUBOT_AWS_REGIONS
#   HUBOT_AWS_KEYNAME
#   HUBOT_AWS_HOSTZONE
#
# Commands:
#   hubot stack delete [stackname]
#   hubot stack create stackname=xxx,template=xxxx - Returns the stack information
#
_ = require 'underscore'
aws = require 'aws2js'
defaultServers = 'proxy,webap,redis,mysql'
defaultCoudFormationConsoleUrl = 'https://console.aws.amazon.com/cloudformation/home'

deleteStack = (msg) ->

    region = process.env.HUBOT_AWS_REGIONS ? 'ap-northeast-1'
    key = process.env.HUBOT_AWS_ACCESS_KEY_ID
    secret = process.env.HUBOT_AWS_SECRET_ACCESS_KEY
    stackname = msg.match[2].replace /^\s+/g, ''

    unless stackname
        msg.send "(failed)\nRequired StackName. Please check your StackName.\n Usage: stack describe"
        return

    cf = aws
        .load('cloudformation', key, secret)
        .setRegion(region)

    cf.request 'DeleteStack', {"StackName": stackname}, (error, reservations) ->
        if error?
            msg.send "(failed)\nFailed to Delete Stack - #{error}"
            return

        msg.send "Please wait until the process is complete."

        # 10秒毎statusを確認する
        statusCheck = []
        intervalId = setInterval ->
            # check ping
            # msg.send '.'
            console.log "."
            switch statusCheck[0]
                when "DELETE_FAILED"
                    clearInterval intervalId
                    msg.reply "(failed)\nFailed to Delete Stack. Please Check the AWS console.\n#{defaultCoudFormationConsoleUrl}?region=#{region}#/stacks?filter=active&stackId=#{stackId}&tab=events"
                    return

            cf.request 'DescribeStacks', (error, reservations) ->
                if error?
                    clearInterval intervalId
                    msg.send "(failed)\nFailed to describe stacks - #{error}"
                    return

                stacks = _.flatten [reservations?.DescribeStacksResult?.Stacks?.member ? []]
                filterd = _.filter stacks, (stack) ->
                    stack.StackName == stackname

                # スタックリストに該当スタック名が存在しない場合、削除完了とみなす
                if filterd.length == 0
                    clearInterval intervalId
                    msg.reply "(successful) DELETE COMPLETE."
                    return

                statusCheck = _.pluck filterd, 'StackStatus'
                stackId = _.pluck filterd, 'StackId'
        , 10000

createStack = (msg) ->

    template = "development"
    environment = 'development'
    region = process.env.HUBOT_AWS_REGIONS ? 'ap-northeast-1'
    key = process.env.HUBOT_AWS_ACCESS_KEY_ID
    secret = process.env.HUBOT_AWS_SECRET_ACCESS_KEY
    keyname = process.env.HUBOT_AWS_KEYNAME
    hostedzone = process.env.HUBOT_AWS_HOSTZONE

    if msg.match[2]?
        params = msg.match[2].replace(/^\s+/g, '').split(',')

    for i, v of params
        param = v.split '='
        switch param[0]
            when "env","e" then environment = param[1].replace /^\s+/g, ''
            when "template","t" then template = param[1].replace /^\s+/g, ''
            when "stackname","n"
                stackname = param[1].replace /^\s+/g, ''
                unless stackname
                    msg.send "(failed) Invalid argument - stackname."
                    return
            else
                msg.send "(failed) Invalid argument. - Usage: env , template , stackname."
                return

    unless stackname?
        msg.send "(failed) Invalid argument - Usage: stackname."
        return

    ec2 = aws
        .load('ec2', key, secret)
        .setApiVersion('2012-05-01')
        .setRegion(region)

    ec2.request 'DescribeImages', {'Owner.1': 'self'}, (error, reservations) ->
        if error?
            msg.send  "(failed) Failed to describe images - #{error}"
            return

        images = _.flatten [reservations?.imagesSet?.item ? []]
        if images.length is 0
            msg.send "No Images..."
            return

        # 各サーバーの最新AMI(imageId)を取得
        imageIds = {}
        for server in defaultServers.split ','
            imageId = getLatestImageId images, "#{environment}-#{server}"
            imageIds[server] = imageId

        query =
            "StackName": stackname
            "TemplateURL": "https://s3-#{region}.amazonaws.com/template-#{environment}-mysite/#{template}.json"
            "Parameters.member.1.ParameterKey": "KeyName"
            "Parameters.member.1.ParameterValue": keyname
            "Parameters.member.2.ParameterKey": "HostedZone"
            "Parameters.member.2.ParameterValue": hostedzone
            "Parameters.member.3.ParameterKey": "UserPrefix"
            "Parameters.member.3.ParameterValue": stackname
            "Parameters.member.4.ParameterKey": "Environment"
            "Parameters.member.4.ParameterValue": environment

        # 各サーバーの最新AMI(imageId)を設定
        i = 5
        for k, v of imageIds
            query['Parameters.member.' + i + '.ParameterKey'] = 'ImageId' + k
            query['Parameters.member.' + i + '.ParameterValue'] = v
            i++

        cf = aws
            .load('cloudformation', key, secret)
            .setRegion(region)

        createStackAndNotifi cf, query, msg

createStackAndNotifi = (cf, query, msg) ->

    cf.request 'CreateStack', query, (error, reservations) ->
        if error?
            msg.send "(failed)\nFailed to Create Stack - #{error}"
            return

        msg.send "Please wait until the process is complete."

        # stackのstatusが[CREATE_COMPLETE]になるまで待機
        # 10秒毎statusを確認する
        statusCheck = ''
        stackMember = ''
        intervalId = setInterval ->
            # check a ping
            # msg.send '.'
            console.log "."
            switch statusCheck
                when "CREATE_COMPLETE"
                    clearInterval intervalId
                    stackInfo = '(successful)\n'
                    for output in stackMember.Outputs.member
                        stackInfo += output.OutputKey + ": " + output.OutputValue + "\n"
                    msg.reply stackInfo
                    return
                when "CREATE_FAILED", "ROLLBACK_COMPLETE", "ROLLBACK_IN_PROGRESS"
                    clearInterval intervalId
                    msg.reply "(failed)\nFailed to Create Stack. Please Check the AWS console.\n#{defaultCoudFormationConsoleUrl}?region=ap-northeast-1#/stacks?filter=active&stackId=#{stackId}&tab=events"
                    return

            cf.request 'DescribeStacks', {'StackName': query.StackName}, (error, reservations) ->
                if error?
                    clearInterval intervalId
                    msg.send "(failed)\nFailed to describe stacks - #{error}"
                    return
                stackMember = reservations.DescribeStacksResult.Stacks.member
                statusCheck = stackMember.StackStatus
                stackId = stackMember.StackId
        , 10000

# AMI名は[環境 + 用途 + 生成日時] e.g.) development-proxy-20130825153141
getLatestImageId = (images, prefix) ->
    amiList = _.filter images, (image) ->
        pos = image.name.indexOf prefix
        return pos >= 0
    num = 0
    imageId = ''
    for ami in amiList
        name = ami.name
        createDate = name.substr prefix.length + 1
        if num < createDate
            num = createDate
            imageId = ami.imageId
    return imageId

module.exports = (robot) ->

    robot.respond /stack( create)( (.+))/i, (msg) ->
        createStack msg

    robot.respond /stack( delete)( (.+))/i, (msg) ->
        deleteStack msg

mysite.json

{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Description": "vpc stack",
  "Parameters": {
    "KeyName": {
      "Description": "Name of an existing EC2 KeyPair to enable SSH access to the instances",
      "Type": "String",
      "MinLength": "1",
      "MaxLength": "64",
      "AllowedPattern": "[-_ a-zA-Z0-9]*",
      "ConstraintDescription": "can contain only alphanumeric characters, spaces, dashes and underscores."
    },
    "UserPrefix": {
      "Description": "Name of Prefix EC2 Instance",
      "Type": "String",
      "MinLength": "1",
      "MaxLength": "64",
      "AllowedPattern": "[-_ a-zA-Z0-9]*",
      "ConstraintDescription": "can contain only alphanumeric characters, spaces, dashes and underscores."
    },
    "Environment": {
      "Description": "Environment of EC2 Instance",
      "Type": "String",
      "MinLength": "1",
      "MaxLength": "64",
      "AllowedPattern": "[-_ a-zA-Z0-9]*",
      "ConstraintDescription": "can contain only alphanumeric characters, spaces, dashes and underscores."
    },
    "HostedZone" : {
      "Description" : "The DNS name of an existing Amazon Route 53 hosted zone",
      "Type" : "String"
    },
    "ImageIdproxy" : {
      "Description" : "The ImageId of ProxyServer latest ImageId",
      "Type" : "String",
      "AllowedPattern": "[-_ a-zA-Z0-9]*",
      "ConstraintDescription": "can contain only alphanumeric characters, spaces, dashes and underscores."
    },
    "ImageIdwebap" : {
      "Description" : "The ImageId of WebServer latest ImageId",
      "Type" : "String",
      "AllowedPattern": "[-_ a-zA-Z0-9]*",
      "ConstraintDescription": "can contain only alphanumeric characters, spaces, dashes and underscores."
    },
    "ImageIdredis" : {
      "Description" : "The ImageId of RedisServer latest ImageId",
      "Type" : "String",
      "AllowedPattern": "[-_ a-zA-Z0-9]*",
      "ConstraintDescription": "can contain only alphanumeric characters, spaces, dashes and underscores."
    },
    "ImageIdmysql" : {
      "Description" : "The ImageId of MySQLServer latest ImageId",
      "Type" : "String",
      "AllowedPattern": "[-_ a-zA-Z0-9]*",
      "ConstraintDescription": "can contain only alphanumeric characters, spaces, dashes and underscores."
    }
  },
  "Resources": {
    "ProxyServer": {
      "Type": "AWS::EC2::Instance",
      "Properties": {
        "DisableApiTermination": "FALSE",
        "ImageId": {"Ref": "ImageIdproxy"},
        "InstanceType": "t1.micro",
        "KeyName": {"Ref": "KeyName"},
        "Monitoring": "true",
        "Tags": [
          {
            "Key": "Name",
            "Value": {"Fn::Join" : [ "", [{"Ref" : "UserPrefix"}, "-proxy-", {"Ref": "Environment"} ]]}
          }
        ],
        "NetworkInterfaces": [
          {
            "DeleteOnTermination": "true",
            "DeviceIndex": 0,
            "SubnetId": "subnet-abc98765",
            "GroupSet": [
              {"Ref": "SecurityGroupSSH"},
              {"Ref": "SecurityGroupHttp"},
              {"Ref": "SecurityGroupFront"},
              "sg-12345678"
            ]
          }
        ],
        "UserData": {"Fn::Base64": {"Fn::Join": ["", [
          "#!/bin/bash -ex\n",
          "exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n",
          "# Configre nginx virtualhost\n",
          "sed -i \"s/SUB_DOMAIN/",
          {"Ref": "UserPrefix"},
          "/g\" /etc/nginx/conf.d/mysite.conf\n",
          "sed -i \"s/WEB_SERVER_IP/",
          {"Fn::GetAtt" : [ "WebServer", "PrivateIp" ]},
          "/g\" /etc/nginx/conf.d/mysite.conf\n",
          "service nginx restart\n"
        ]]}}
      }
    },
    "WebServer": {
      "Type": "AWS::EC2::Instance",
      "Properties": {
        "DisableApiTermination": "FALSE",
        "ImageId": {"Ref": "ImageIdwebap"},
        "InstanceType": "m1.small",
        "KeyName": {"Ref": "KeyName"},
        "Monitoring": "true",
        "Tags": [
          {
            "Key": "Name",
            "Value": {"Fn::Join" : [ "", [{"Ref" : "UserPrefix"}, "-web-", {"Ref": "Environment"} ]]}
          }
        ],
        "NetworkInterfaces": [
          {
            "DeleteOnTermination": "true",
            "DeviceIndex": 0,
            "SubnetId": "subnet-abc12345",
            "GroupSet": [
              {"Ref": "SecurityGroupMiddle"},
              "sg-12345678"
            ]
          }
        ],
        "UserData": {"Fn::Base64": {"Fn::Join": ["", [
          "#!/bin/bash\n",
          "exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n",
          "# Configre fuelphp db.php\n",
          "sed -i \"s/MYSQL_SERVER_IP/",
          {"Fn::GetAtt" : [ "MySQLServer", "PrivateIp" ]},
          "/g\" /opt/mysite/fuel/app/config/",
          {"Ref": "Environment"},
          "/db.php\n",
          "sed -i \"s/REDIS_SERVER_IP/",
          {"Fn::GetAtt" : [ "RedisServer", "PrivateIp" ]},
          "/g\" /opt/mysite/fuel/app/config/",
          {"Ref": "Environment"},
          "/db.php\n"
        ]]}}
      }
    },
    "RedisServer": {
      "Type": "AWS::EC2::Instance",
      "Properties": {
        "DisableApiTermination": "FALSE",
        "ImageId": {"Ref": "ImageIdredis"},
        "InstanceType": "m1.small",
        "KeyName": {"Ref": "KeyName"},
        "Monitoring": "true",
        "Tags": [
          {
            "Key": "Name",
            "Value": {"Fn::Join" : [ "", [{"Ref" : "UserPrefix"}, "-redis-", {"Ref": "Environment"} ]]}
          }
        ],
        "NetworkInterfaces": [
          {
            "DeleteOnTermination": "true",
            "DeviceIndex": 0,
            "SubnetId": "subnet-abc12345",
            "GroupSet": [
              {"Ref": "SecurityGroupBack"},
              "sg-12345678"
            ]
          }
        ]
      }
    },
    "MySQLServer": {
      "Type": "AWS::EC2::Instance",
      "Properties": {
        "DisableApiTermination": "FALSE",
        "ImageId": {"Ref": "ImageIdmysql"},
        "InstanceType": "m1.small",
        "KeyName": {"Ref": "KeyName"},
        "Monitoring": "true",
        "Tags": [
          {
            "Key": "Name",
            "Value": {"Fn::Join" : [ "", [{"Ref" : "UserPrefix"}, "-mysql-", {"Ref": "Environment"} ]]}
          }
        ],
        "NetworkInterfaces": [
          {
            "DeleteOnTermination": "true",
            "DeviceIndex": 0,
            "SubnetId": "subnet-abc12345",
            "GroupSet": [
              {"Ref": "SecurityGroupBack"},
              "sg-12345678"
            ]
          }
        ]
      }
    },
    "DNSRecord" : {
      "Type" : "AWS::Route53::RecordSet",
      "Properties" : {
        "HostedZoneName" : {"Fn::Join" : [ "", [{"Ref" : "HostedZone"}, "." ]]},
        "Comment" : "A record for the Bastion instance.",
        "Name" : { "Fn::Join" : [ "", [{"Ref" : "UserPrefix"}, "." , {"Ref" : "HostedZone"}, "." ]]},
        "Type" : "A",
        "TTL" : "300",
        "ResourceRecords" : [{"Ref" :"EIPProxyServer"}]
      }
    },
    "EIPProxyServer": {
      "Type": "AWS::EC2::EIP",
      "Properties": {
        "Domain": "vpc",
        "InstanceId": {"Ref": "ProxyServer"}
      }
    },
    "SecurityGroupFront": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription" : "Proxy Server",
        "VpcId": "vpc-12345678"
      }
    },
    "SecurityGroupMiddle": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription" : "Application Server",
        "VpcId": "vpc-12345678"
      }
    },
    "SecurityGroupBack": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Storage Server",
        "VpcId": "vpc-12345678"
      }
    },
    "SecurityGroupSSH": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Allow ssh from proxy server",
        "VpcId": "vpc-12345678",
        "SecurityGroupIngress": [
          {
            "IpProtocol": "tcp",
            "CidrIp": "12.345.678.90/32",
            "FromPort": "22",
            "ToPort": "22"
          }
        ]
      }
    },
    "SecurityGroupHttp": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Allow http from CA",
        "VpcId": "vpc-12345678"
      }
    },
    "SecurityGroupIngress1" :{
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties" : {
        "GroupId": { "Ref": "SecurityGroupBack" },
        "IpProtocol": "tcp",
        "FromPort": "3306",
        "ToPort": "3306",
        "SourceSecurityGroupId": { "Ref": "SecurityGroupMiddle" }
      }
    },
    "SecurityGroupIngress2" :{
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties" : {
        "GroupId": { "Ref": "SecurityGroupBack" },
        "IpProtocol": "tcp",
        "FromPort": "6379",
        "ToPort": "6379",
        "SourceSecurityGroupId": { "Ref": "SecurityGroupMiddle" }
      }
    },
    "SecurityGroupIngress3" :{
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties" : {
        "GroupId": { "Ref": "SecurityGroupMiddle" },
        "IpProtocol": "tcp",
        "FromPort": "80",
        "ToPort": "80",
        "SourceSecurityGroupId": { "Ref": "SecurityGroupFront" }
      }
    },
    "SecurityGroupIngress4" :{
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties" : {
        "GroupId": { "Ref": "SecurityGroupHttp" },
        "IpProtocol": "tcp",
        "FromPort": "443",
        "ToPort": "443",
        "CidrIp": "12.345.678.90/32"
      }
    },
    "SecurityGroupIngress5" :{
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties" : {
        "GroupId": { "Ref": "SecurityGroupBack" },
        "IpProtocol": "tcp",
        "FromPort": "22",
        "ToPort": "22",
        "SourceSecurityGroupId": { "Ref": "SecurityGroupFront" }
      }
    },
    "SecurityGroupIngress6" :{
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties" : {
        "GroupId": { "Ref": "SecurityGroupMiddle" },
        "IpProtocol": "tcp",
        "FromPort": "22",
        "ToPort": "22",
        "SourceSecurityGroupId": { "Ref": "SecurityGroupFront" }
      }
    }
  },
  "Outputs" : {
    "ApplicationURL" : {
      "Description" : "URL of running web application",
      "Value" : {"Fn::Join" : [ "", ["https://", {"Ref" : "UserPrefix"}, "." , {"Ref" : "HostedZone"} ]]}
    },
    "AvailabilityZone" : {
      "Description" : "AvailabilityZone of EC2 Instance",
      "Value" : {"Fn::GetAtt" : [ "ProxyServer", "AvailabilityZone" ]}
    },
    "ProxyServerEIP" : {
      "Description" : "EIP of ProxyServer",
      "Value" : {"Ref" : "EIPProxyServer"}
    },
    "ProxyServerIpAddress" : {
      "Description" : "IP Address of ProxyServer",
      "Value" : {"Fn::GetAtt" : [ "ProxyServer", "PrivateIp" ]}
    },
    "WebServerIpAddress" : {
      "Description" : "IP Address of WebServer",
      "Value" : {"Fn::GetAtt" : [ "WebServer", "PrivateIp" ]}
    },
    "RedisServerIpAddress" : {
      "Description" : "IP Address of RedisServer",
      "Value" : {"Fn::GetAtt" : [ "RedisServer", "PrivateIp" ]}
    },
    "MySQLServerIpAddress" : {
      "Description" : "IP Address of MySQLServer",
      "Value" : {"Fn::GetAtt" : [ "MySQLServer", "PrivateIp" ]}
    },
    "SSHToProxyServer" : {
      "Value" : { "Fn::Join" :["", [
        "ssh -i /path/to/", {"Ref" : "KeyName"}, ".pem",
        " ec2-user@", {"Ref" : "EIPProxyServer"}
      ]] }, 
      "Description" : "SSH command to connect ProxyServer"
    }
  }
}