Created
January 3, 2016 19:21
-
-
Save davidski/cdf98e29f49520de0a8e to your computer and use it in GitHub Desktop.
Elasticsearch-Logstash-Kibana Opsworks stack creation via CloudFormation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "AWSTemplateFormatVersion": "2010-09-09", | |
| "Description": "Create the ELK OpsWorks Stack.", | |
| "Metadata": { | |
| "AWS::CloudFormation::Interface": { | |
| "ParameterGroups": [{ | |
| "Label": { "default": "Network"}, | |
| "Parameters": [ "VpcID", "SubnetID", "KeyName" ] | |
| }, { | |
| "Label": { "default": "ELK Config"}, | |
| "Parameters": [ "CookbookBundle", "ElasticsearchCluster", "LSInputS3Bucket", "LSInputS3Prefix" ] | |
| }, { | |
| "Label": { "default": "Instance Selection"}, | |
| "Parameters": [ "ElasticsearchInstanceType", "ShipperInstanceType", "RedisInstanceType", "IndexerInstanceType", "KibanaInstanceType" ] | |
| }] | |
| } | |
| }, | |
| "Parameters": { | |
| "AuditlogBucket": { | |
| "Type": "String", | |
| "Default": "BUCKET", | |
| "Description": "Enter the pre-existing bucket to which logs should be sent." | |
| }, | |
| "CookbookBundle": { | |
| "Type": "String", | |
| "Default": "cookbooks/elk-cookbooks.tar.gz", | |
| "Description": "Enter the S3 key of the ELK cookbooks bundle." | |
| }, | |
| "ElasticsearchCluster": { | |
| "Type": "String", | |
| "Default": "elktest", | |
| "Description": "Enter the name of the Elasticsearch cluster." | |
| }, | |
| "ElasticsearchVersion": { | |
| "Type": "String", | |
| "Default": "2.1.1", | |
| "Description": "Enter the version of Elasticsearch to deploy." | |
| }, | |
| "LSInputS3Bucket": { | |
| "Type": "String", | |
| "Default": "BUCKET", | |
| "Description": "Enter the S3 bucket of the ELK input files." | |
| }, | |
| "LSInputS3Prefix": { | |
| "Type": "String", | |
| "Default": "2-ELK/input", | |
| "Description": "Enter the key prefix to the ELK input files." | |
| }, | |
| "ElasticsearchInstanceType": { | |
| "Type": "String", | |
| "Default": "m3.2xlarge", | |
| "Description": "EC2 instance type to use for Elasticsearch nodes." | |
| }, | |
| "KibanaInstanceType": { | |
| "Type": "String", | |
| "Default": "t2.micro", | |
| "Description": "EC2 instance type to use for Kibana node." | |
| }, | |
| "IndexerInstanceType": { | |
| "Type": "String", | |
| "Default": "c3.2xlarge", | |
| "Description": "EC2 instance type to use for Logstash Indexer nodes." | |
| }, | |
| "ShipperInstanceType": { | |
| "Type": "String", | |
| "Default": "c3.large", | |
| "Description": "EC2 instance type to use for Logstash Shipper nodes." | |
| }, | |
| "RedisInstanceType": { | |
| "Type": "String", | |
| "Default": "r3.large", | |
| "Description": "EC2 instance type to use for Redis nodes." | |
| }, | |
| "KeyName": { | |
| "Description": "The EC2 Key Pair to allow SSH access to the instances.", | |
| "Default": "KEYNAME", | |
| "Type": "AWS::EC2::KeyPair::KeyName", | |
| "ConstraintDescription": "Must be the name of an existing EC2 KeyPair." | |
| }, | |
| "ELK1SubnetID": { | |
| "Type": "AWS::EC2::Subnet::Id", | |
| "Default": "SUBNETID", | |
| "Description": "Enter the subnet ID for the first ELK zone." | |
| }, | |
| "ELK2SubnetID": { | |
| "Type": "AWS::EC2::Subnet::Id", | |
| "Default": "SUBNETID", | |
| "Description": "Enter the subnet ID for the second ELK zone." | |
| }, | |
| "ELK3SubnetID": { | |
| "Type": "AWS::EC2::Subnet::Id", | |
| "Default": "SUBNETID", | |
| "Description": "Enter the subnet ID for the third ELK zone." | |
| }, | |
| "SubnetID": { | |
| "Type": "AWS::EC2::Subnet::Id", | |
| "Default": "SUBNETID", | |
| "Description": "Enter the subnet ID for the general ELK zone." | |
| }, | |
| "VpcID": { | |
| "Type": "String", | |
| "Default": "VPCID", | |
| "Description": "Enter the VPC ID in which to run." | |
| }, | |
| "InfrastructureBucket": { | |
| "Type": "String", | |
| "Default": "BUCKET", | |
| "Description": "Enter the pre-existing bucket for infrastructure code." | |
| }, | |
| "LambdaCode": { | |
| "Type": "String", | |
| "Default": "lambda/opsworks_instance.zip", | |
| "Description": "Enter the location of the Lambda custom opsworks function." | |
| } | |
| }, | |
| "Resources": { | |
| "OpsworksInstanceFunction": { | |
| "Type": "AWS::Lambda::Function", | |
| "Properties": { | |
| "Code": { | |
| "S3Bucket": { "Ref": "InfrastructureBucket" }, | |
| "S3Key": { "Ref": "LambdaCode" } | |
| }, | |
| "Handler": "opsworks_instance.lambda_handler", | |
| "Runtime": "python2.7", | |
| "Timeout": "30", | |
| "Role": { "Fn::GetAtt" : ["LambdaExecutionRole", "Arn"] } | |
| } | |
| }, | |
| "LambdaExecutionRole": { | |
| "Type": "AWS::IAM::Role", | |
| "Properties": { | |
| "AssumeRolePolicyDocument": { | |
| "Version": "2012-10-17", | |
| "Statement": [{ | |
| "Effect": "Allow", | |
| "Principal": {"Service": ["lambda.amazonaws.com"]}, | |
| "Action": ["sts:AssumeRole"] | |
| }] | |
| }, | |
| "Path": "/", | |
| "ManagedPolicyArns": [ | |
| "arn:aws:iam::aws:policy/AWSOpsWorksFullAccess" | |
| ], | |
| "Policies": [{ | |
| "PolicyName": "root", | |
| "PolicyDocument": { | |
| "Version": "2012-10-17", | |
| "Statement": [{ | |
| "Effect": "Allow", | |
| "Action": ["logs:CreateLogGroup","logs:CreateLogStream","logs:PutLogEvents"], | |
| "Resource": "arn:aws:logs:*:*:*" | |
| }] | |
| } | |
| }] | |
| } | |
| }, | |
| "myStack": { | |
| "Type": "AWS::OpsWorks::Stack", | |
| "Properties": { | |
| "DefaultInstanceProfileArn": "ARN", | |
| "DefaultSshKeyName": { "Ref": "KeyName" }, | |
| "DefaultSubnetId": { "Ref": "SubnetID" }, | |
| "DefaultOs": "Amazon Linux 2015.09", | |
| "ConfigurationManager": { | |
| "Name": "Chef", | |
| "Version": "12" | |
| }, | |
| "VpcId": { "Ref": "VpcID" }, | |
| "Name": {"Fn::Join": ["-", [{"Ref": "AWS::StackName"}, "ELK"]]}, | |
| "ServiceRoleArn": "ARN", | |
| "UseCustomCookbooks": "true", | |
| "UseOpsworksSecurityGroups": "false", | |
| "CustomCookbooksSource": { | |
| "Type": "s3", | |
| "Url": { | |
| "Fn::Join": ["", ["https://s3-", | |
| { "Ref": "AWS::Region"}, | |
| ".amazonaws.com/", | |
| { "Ref": "InfrastructureBucket" }, | |
| "/", | |
| { "Ref": "CookbookBundle" } | |
| ]] | |
| } | |
| }, | |
| "DefaultRootDeviceType": "instance-store", | |
| "CustomJson": { | |
| "apt": { | |
| "compile_time_update": "true" | |
| }, | |
| "collectd": { | |
| "version": "5.4.1", | |
| "ulr": "https://s3.amazonaws.com/collectd-5.4.1/collectd-5-4-1.tar.gz", | |
| "graphite_ipaddress": "10.0.0.51", | |
| "plugins": { | |
| "interface": {}, | |
| "write_graphite": { | |
| "config": { | |
| "SeparateInstances": true, | |
| "EscapeCharacter": "_", | |
| "StoreRates": false | |
| } | |
| }, | |
| "cpu": {}, | |
| "load": {}, | |
| "memory": {} | |
| } | |
| }, | |
| "java": { | |
| "install_flavor": "oracle", | |
| "jdk_version": "8", | |
| "oracle": { | |
| "accept_oracle_download_terms": true | |
| } | |
| }, | |
| "redisio": { | |
| "version": "2.8.21", | |
| "servers": [{ | |
| "port": "6379", | |
| "maxmemory": "85%", | |
| "save": "" | |
| }] | |
| }, | |
| "sysctl": { | |
| "params": { | |
| "vm": { | |
| "overcommit_memory": "1" | |
| } | |
| } | |
| }, | |
| "logstash": { | |
| "xms": "500M", | |
| "config_templates_cookbook": "sch-logstash", | |
| "instance_default": { | |
| "maxmind_license_key": "MAXMIND_KEY" | |
| }, | |
| "plugin_path": "false", | |
| "templates": { | |
| "input_s3_bucket": {"Ref": "LSInputS3Bucket"}, | |
| "input_s3_prefix": {"Ref": "LSInputS3Prefix"}, | |
| "redis_congestion_threshold": 25000000 | |
| } | |
| }, | |
| "kibana": { | |
| "webserver": "apache", | |
| "webserver_hostname": "HOSTNAME", | |
| "webserver_listen": "*", | |
| "es_server": "localhost", | |
| "config": { | |
| "kibana_index": ".kibana4" | |
| } | |
| }, | |
| "graphite": { | |
| "listen_port": 80, | |
| "url": "HOSTNAME" | |
| }, | |
| "elasticsearch-curator": { | |
| "elasticsearch_server": "localhost", | |
| "days_to_keep": "20", | |
| "optimize_indices_after": "5", | |
| "snapshot_repository": "elk_snapshots", | |
| "backup_weekday": "6", | |
| "dekete_snapshots_older_than": "60", | |
| "bin_dir": "/usr/local/bin/" | |
| }, | |
| "elasticsearch": { | |
| "version": {"Ref": "ElasticsearchVersion"}, | |
| "config": { | |
| "cluster.name": {"Ref": "ElasticsearchCluster"}, | |
| "network.publish_host": "_non_loopback:ipv4_", | |
| "network.bind_host": "0.0.0.0" | |
| }, | |
| "plugins": { | |
| "kopf": "lmenezes/elasticsearch-kopf/2.0", | |
| "cloud-aws": "cloud-aws" | |
| }, | |
| "cloud": { | |
| "aws": { | |
| "region": "us-west-2" | |
| }, | |
| "node": { | |
| "autoattributes": "true" | |
| } | |
| }, | |
| "discovery": { | |
| "zen": { | |
| "minimum_master_nodes": 2 | |
| }, | |
| "type": "ec2", | |
| "ec2": { | |
| "host_type": "private_ip" | |
| } | |
| }, | |
| "data": { | |
| "devices": { | |
| "/dev/md0": { | |
| "file_system": "ext3", | |
| "mount_options": "rw,user", | |
| "mount_path": "/es", | |
| "format_command": "mkfs.ext3", | |
| "fs_check_command": "dumpe2fs" | |
| } | |
| } | |
| }, | |
| "index": { | |
| "number_of_replicas": "1", | |
| "number_of_shards": "2" | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "myESLayer": { | |
| "Type": "AWS::OpsWorks::Layer", | |
| "Properties": { | |
| "StackId": { | |
| "Ref": "myStack" | |
| }, | |
| "Type": "custom", | |
| "Shortname": "es", | |
| "EnableAutoHealing": "true", | |
| "AutoAssignElasticIps": "false", | |
| "AutoAssignPublicIps": "false", | |
| "Name": "Elasticsearch", | |
| "CustomSecurityGroupIds": ["SECURITYGROUP"], | |
| "CustomRecipes": { | |
| "Undeploy": [], | |
| "Setup": [], | |
| "Configure": [], | |
| "Shutdown": [], | |
| "Deploy": [ | |
| "apt::default", | |
| "sch-base::default", | |
| "sch-user::default", | |
| "sch-collectd::default", | |
| "java::default", | |
| "sch-elasticsearch::create_raid", | |
| "sch-elasticsearch::data", | |
| "sch-elasticsearch::default" | |
| ] | |
| } | |
| } | |
| }, | |
| "myLSShipperLayer": { | |
| "Type": "AWS::OpsWorks::Layer", | |
| "Properties": { | |
| "StackId": { | |
| "Ref": "myStack" | |
| }, | |
| "Type": "custom", | |
| "Shortname": "ls_shipper", | |
| "EnableAutoHealing": "true", | |
| "AutoAssignElasticIps": "false", | |
| "AutoAssignPublicIps": "false", | |
| "Name": "Logstash Shipper", | |
| "CustomSecurityGroupIds": ["SECURITYGROUP"], | |
| "CustomRecipes": { | |
| "Undeploy": [], | |
| "Setup": [], | |
| "Configure": [], | |
| "Shutdown": [], | |
| "Deploy": [ | |
| "apt::default", | |
| "sch-base::default", | |
| "sch-user::default", | |
| "sch-collectd::default", | |
| "java::default", | |
| "sch-logstash::shipper" | |
| ] | |
| } | |
| } | |
| }, | |
| "myLSIndexerLayer": { | |
| "Type": "AWS::OpsWorks::Layer", | |
| "Properties": { | |
| "StackId": { | |
| "Ref": "myStack" | |
| }, | |
| "Type": "custom", | |
| "Shortname": "ls_indexer", | |
| "EnableAutoHealing": "true", | |
| "AutoAssignElasticIps": "false", | |
| "AutoAssignPublicIps": "false", | |
| "Name": "Logstash Indexer", | |
| "CustomSecurityGroupIds": ["SECURITYGROUP"], | |
| "CustomRecipes": { | |
| "Undeploy": [], | |
| "Setup": [], | |
| "Configure": [], | |
| "Shutdown": [], | |
| "Deploy": [ | |
| "apt::default", | |
| "sch-base::default", | |
| "sch-user::default", | |
| "sch-collectd::default", | |
| "java::default", | |
| "sch-logstash::indexer" | |
| ] | |
| } | |
| } | |
| }, | |
| "myRedisLayer": { | |
| "Type": "AWS::OpsWorks::Layer", | |
| "Properties": { | |
| "StackId": { | |
| "Ref": "myStack" | |
| }, | |
| "Type": "custom", | |
| "Shortname": "redis", | |
| "EnableAutoHealing": "true", | |
| "AutoAssignElasticIps": "false", | |
| "AutoAssignPublicIps": "false", | |
| "Name": "Redis Buffer", | |
| "CustomSecurityGroupIds": ["SECURITYGROUP"], | |
| "CustomRecipes": { | |
| "Undeploy": [], | |
| "Setup": [], | |
| "Configure": [], | |
| "Shutdown": [], | |
| "Deploy": [ | |
| "apt::default", | |
| "sch-base::default", | |
| "sch-user::default", | |
| "sch-collectd::default", | |
| "sysctl::default", | |
| "redisio::install", | |
| "redisio::enable", | |
| "sch-redis-monitor::default" | |
| ] | |
| } | |
| } | |
| }, | |
| "myKibanaLayer": { | |
| "Type": "AWS::OpsWorks::Layer", | |
| "Properties": { | |
| "StackId": { | |
| "Ref": "myStack" | |
| }, | |
| "Type": "custom", | |
| "Shortname": "kb", | |
| "EnableAutoHealing": "true", | |
| "AutoAssignElasticIps": "false", | |
| "AutoAssignPublicIps": "false", | |
| "Name": "Kibana", | |
| "CustomSecurityGroupIds": ["SECURITYGROUP"], | |
| "CustomRecipes": { | |
| "Undeploy": [], | |
| "Setup": [], | |
| "Configure": [], | |
| "Shutdown": [], | |
| "Deploy": [ | |
| "apt::default", | |
| "sch-base::default", | |
| "sch-user::default", | |
| "python::default", | |
| "es-curator::default", | |
| "java::default", | |
| "sch-elasticsearch::client_node", | |
| "sch-kibana::default" | |
| ] | |
| } | |
| } | |
| }, | |
| "myES1Instance": { | |
| "Type": "Custom::OpsworksInstance", | |
| "Properties": { | |
| "ServiceToken": { | |
| "Fn::GetAtt" : [ | |
| "OpsworksInstanceFunction", "Arn" | |
| ] | |
| }, | |
| "EbsOptimized": false, | |
| "InstallUpdatesOnBoot": true, | |
| "InstanceType": { "Ref": "ElasticsearchInstanceType" }, | |
| "LayerId": { "Ref": "myESLayer" }, | |
| "RootDeviceType": "instance-store", | |
| "StackId": { "Ref": "myStack" }, | |
| "SubnetId": { "Ref": "ELK1SubnetID" } | |
| } | |
| }, | |
| "myES2Instance": { | |
| "Type": "Custom::OpsworksInstance", | |
| "Properties": { | |
| "ServiceToken": { | |
| "Fn::GetAtt" : [ | |
| "OpsworksInstanceFunction", "Arn" | |
| ] | |
| }, | |
| "EbsOptimized": false, | |
| "InstallUpdatesOnBoot": true, | |
| "InstanceType": { "Ref": "ElasticsearchInstanceType" }, | |
| "LayerId": { "Ref": "myESLayer" }, | |
| "RootDeviceType": "instance-store", | |
| "StackId": { "Ref": "myStack" }, | |
| "SubnetId": { "Ref": "ELK2SubnetID" } | |
| } | |
| }, | |
| "myES3Instance": { | |
| "Type": "Custom::OpsworksInstance", | |
| "Properties": { | |
| "ServiceToken": { | |
| "Fn::GetAtt" : [ | |
| "OpsworksInstanceFunction", "Arn" | |
| ] | |
| }, | |
| "EbsOptimized": false, | |
| "InstallUpdatesOnBoot": true, | |
| "InstanceType": { "Ref": "ElasticsearchInstanceType" }, | |
| "LayerId": { "Ref": "myESLayer" }, | |
| "RootDeviceType": "instance-store", | |
| "StackId": { "Ref": "myStack" }, | |
| "SubnetId": { "Ref": "ELK3SubnetID" } | |
| } | |
| }, | |
| "myES4Instance": { | |
| "Type": "Custom::OpsworksInstance", | |
| "Properties": { | |
| "ServiceToken": { | |
| "Fn::GetAtt" : [ | |
| "OpsworksInstanceFunction", "Arn" | |
| ] | |
| }, | |
| "EbsOptimized": false, | |
| "InstallUpdatesOnBoot": true, | |
| "InstanceType": { "Ref": "ElasticsearchInstanceType" }, | |
| "LayerId": { "Ref": "myESLayer" }, | |
| "RootDeviceType": "instance-store", | |
| "StackId": { "Ref": "myStack" }, | |
| "SubnetId": { "Ref": "ELK3SubnetID" } | |
| } | |
| }, | |
| "myLSi1Instance": { | |
| "Type": "Custom::OpsworksInstance", | |
| "Properties": { | |
| "ServiceToken": { | |
| "Fn::GetAtt" : [ | |
| "OpsworksInstanceFunction", "Arn" | |
| ] | |
| }, | |
| "EbsOptimized": false, | |
| "InstallUpdatesOnBoot": true, | |
| "InstanceType": { "Ref": "IndexerInstanceType" }, | |
| "LayerId": { "Ref": "myLSIndexerLayer" }, | |
| "RootDeviceType": "instance-store", | |
| "StackId": { "Ref": "myStack" } | |
| } | |
| }, | |
| "myLSi2Instance": { | |
| "Type": "Custom::OpsworksInstance", | |
| "Properties": { | |
| "ServiceToken": { | |
| "Fn::GetAtt" : [ | |
| "OpsworksInstanceFunction", "Arn" | |
| ] | |
| }, | |
| "EbsOptimized": false, | |
| "InstallUpdatesOnBoot": true, | |
| "InstanceType": { "Ref": "IndexerInstanceType" }, | |
| "LayerId": { "Ref": "myLSIndexerLayer" }, | |
| "RootDeviceType": "instance-store", | |
| "StackId": { "Ref": "myStack" } | |
| } | |
| }, | |
| "myLSi3Instance": { | |
| "Type": "Custom::OpsworksInstance", | |
| "Properties": { | |
| "ServiceToken": { | |
| "Fn::GetAtt" : [ | |
| "OpsworksInstanceFunction", "Arn" | |
| ] | |
| }, | |
| "EbsOptimized": false, | |
| "InstallUpdatesOnBoot": true, | |
| "InstanceType": { "Ref": "IndexerInstanceType" }, | |
| "LayerId": { "Ref": "myLSIndexerLayer" }, | |
| "RootDeviceType": "instance-store", | |
| "StackId": { "Ref": "myStack" } | |
| } | |
| }, | |
| "myLSs1Instance": { | |
| "Type": "Custom::OpsworksInstance", | |
| "Properties": { | |
| "ServiceToken": { | |
| "Fn::GetAtt" : [ | |
| "OpsworksInstanceFunction", "Arn" | |
| ] | |
| }, | |
| "EbsOptimized": false, | |
| "InstallUpdatesOnBoot": true, | |
| "InstanceType": { "Ref": "ShipperInstanceType" }, | |
| "LayerId": { "Ref": "myLSShipperLayer" }, | |
| "RootDeviceType": "instance-store", | |
| "StackId": { "Ref": "myStack" } | |
| } | |
| }, | |
| "myKibanaInstance": { | |
| "Type": "Custom::OpsworksInstance", | |
| "Properties": { | |
| "ServiceToken": { | |
| "Fn::GetAtt" : [ | |
| "OpsworksInstanceFunction", "Arn" | |
| ] | |
| }, | |
| "EbsOptimized": false, | |
| "InstallUpdatesOnBoot": true, | |
| "InstanceType": { "Ref": "KibanaInstanceType" }, | |
| "LayerId": { "Ref": "myKibanaLayer" }, | |
| "RootDeviceType": "ebs", | |
| "StackId": { "Ref": "myStack" } | |
| } | |
| }, | |
| "myRedisInstance": { | |
| "Type": "Custom::OpsworksInstance", | |
| "Properties": { | |
| "ServiceToken": { | |
| "Fn::GetAtt" : [ | |
| "OpsworksInstanceFunction", "Arn" | |
| ] | |
| }, | |
| "EbsOptimized": false, | |
| "InstallUpdatesOnBoot": true, | |
| "InstanceType": { "Ref": "RedisInstanceType" }, | |
| "LayerId": { "Ref": "myRedisLayer" }, | |
| "RootDeviceType": "instance-store", | |
| "StackId": { "Ref": "myStack" } | |
| } | |
| } | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment