/cephfs.json

## cephfs.json
{
  "AWSTemplateFormatVersion" : "2010-09-09",

  "Description" : "CephFS CloudFormation template.",

  "Parameters" : {
    "DataNodeCount" : {
      "Description" : "Number of data nodes to provision in cluster (2-18)",
      "Type" : "Number",
      "Default" : "2",
      "MinValue" : "2",
      "MaxValue" : "18",
      "ConstraintDescription" : "must be between 2 and 18."
    },
    "KeyPair": {
      "Description" : "Name of an existing EC2 KeyPair: https://console.aws.amazon.com/ec2/v2/home#KeyPairs:",
      "Type": "String",
      "MinLength": "1",
      "MaxLength": "255",
      "AllowedPattern" : "[\\x20-\\x7E]*",
      "ConstraintDescription" : "can contain only ASCII characters."
    },
    "ClusterAccess" : {
      "Description" : "CIDR IP range allowed to login to the cluster instances",
      "Type": "String",
      "MinLength": "9",
      "MaxLength": "18",
      "Default": "0.0.0.0/0",
      "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
      "ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x."
    },
    "ClusterInstanceType" : {
      "Description" : "AMI instance type: https://aws.amazon.com/ec2/instance-types/",
      "Type" : "String",
      "Default" : "m1.medium"
    },
    "GUID" : {
      "Description" : "Globally Unique ID: http://www.guidgen.com/",
      "Type" : "String",
      "MinLength": "36",
      "MaxLength": "36"
    },
    "MountType" : {
      "Description" : "The fuse client is the easiest way to get up to date code, while the kernel client will often give better performance",
      "Type" : "String",
      "Default" : "kernel",
      "AllowedValues": ["kernel", "fuse"]
    }
  },

  "Mappings" : {
    "LinuxRegionMap" : {
      "source"         : { "URL" : "http://cloud-images.ubuntu.com/locator/ec2/  -->  search for '14.04 amd64 instance aki' for details" },
      "ap-northeast-1" : { "AMI" : "ami-a3615bcd" },
      "ap-southeast-1" : { "AMI" : "ami-f6529d95" },
      "eu-central-1"   : { "AMI" : "ami-42f3eb2e" },
      "eu-west-1"      : { "AMI" : "ami-5f8e382c" },
      "sa-east-1"      : { "AMI" : "ami-aa1999c6" },
      "us-east-1"      : { "AMI" : "ami-91c5eafb" },
      "us-west-1"      : { "AMI" : "ami-6f22540f" },
      "cn-north-1"     : { "AMI" : "ami-d67db4bb" },
      "us-gov-west-1"  : { "AMI" : "ami-5cbbd97f" },
      "ap-southeast-2" : { "AMI" : "ami-01dffb62" },
      "us-west-2"      : { "AMI" : "ami-befd1bde" }
    }
  },

  "Resources" : {

    "ClusterSecurityGroup" : {
      "Type" : "AWS::EC2::SecurityGroup",
      "Properties" : {
        "GroupDescription" : "Cluster Security Group",
	"Tags" : [{ "Key" : "Name", "Value" : "Cluster Security Group" }],
        "SecurityGroupIngress" : [
	  {
            "IpProtocol" : "tcp",
            "FromPort" : "22",
            "ToPort" : "22",
            "CidrIp" : { "Ref" : "ClusterAccess" }
          }
        ]
      }
    },

    "UpdateSecurityGroup" : {
      "Type" : "AWS::EC2::SecurityGroupIngress",
      "Properties" : {
        "GroupName" : { "Ref" : "ClusterSecurityGroup" },
	"SourceSecurityGroupName" : { "Ref" : "ClusterSecurityGroup" },
        "IpProtocol" : "tcp",
        "FromPort" : "0",
        "ToPort" : "65535"
      }
    },

    "Master1" : {
      "Type" : "AWS::EC2::Instance",
      "Properties" : {
	"InstanceType" : { "Ref" : "ClusterInstanceType" },
        "SecurityGroups" : [ { "Ref" : "ClusterSecurityGroup" } ],
        "KeyName" : { "Ref" : "KeyPair" },
        "ImageId" : { "Fn::FindInMap" : [ "LinuxRegionMap", { "Ref" : "AWS::Region" }, "AMI" ]},
	"Tags" : [{ "Key" : "Name", "Value" : "CephFS Server" }],
	"UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [
	  "#!/bin/bash\n",
	  "sudo su -\n",
	  "wget -q -O- 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc' | sudo apt-key add -\n",
	  "echo deb http://ceph.com/debian-infernalis/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list\n",
	  "apt-get update -y\n",
	  "apt-get install -y ntp openssh-server ceph-deploy\n",
	  "sudo -u ubuntu bash <<\"EOF\"\n",
	  "cat /dev/zero | ssh-keygen -q -N \"\"\n",
	  "test -f /home/ubuntu/.ssh/id_rsa.pub && cat /home/ubuntu/.ssh/id_rsa.pub >> /home/ubuntu/.ssh/authorized_keys && chmod 644 /home/ubuntu/.ssh/authorized_keys\n",
	  "CURLIO=$( ( gpg --cipher-algo AES256 --symmetric --yes --batch --passphrase=",
	  { "Ref" : "GUID" },
	  " -c /home/ubuntu/.ssh/id_rsa && curl -F \"file=@/home/ubuntu/.ssh/id_rsa.gpg\" https://curl.io/send/v2ioebm0 ) 2>&1 | grep '^https')\n",
	  "PUBLICTOKEN=$( echo -n ",
	  { "Ref" : "GUID" },
	  " | sha1sum | awk '{print $1}' )\n",
	  "test -n ${CURLIO} && ( curl -s \"https://scry.in/api.php?action=shorturl&format=simple&keyword=${PUBLICTOKEN}&url=${CURLIO}\" > /dev/null ) && rm /home/ubuntu/.ssh/id_rsa.gpg\n",
	  "sudo mkdir /var/local/osd1 && sudo chmod 777 /var/local/osd1\n",
	  "touch /tmp/Master1\n",
	  "EOF"
	]]}}
      }
    },

    "Master2" : {
      "Type" : "AWS::EC2::Instance",
      "DependsOn" : "Master1",
      "Properties" : {
	"InstanceType" : { "Ref" : "ClusterInstanceType" },
        "SecurityGroups" : [ { "Ref" : "ClusterSecurityGroup" } ],
        "KeyName" : { "Ref" : "KeyPair" },
        "ImageId" : { "Fn::FindInMap" : [ "LinuxRegionMap", { "Ref" : "AWS::Region" }, "AMI" ]},
	"Tags" : [{ "Key" : "Name", "Value" : "CephFS Server" }],
	"UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [
	  "#!/bin/bash\n",
	  "sudo su -\n",
	  "wget -q -O- 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc' | sudo apt-key add -\n",
	  "echo deb http://ceph.com/debian-infernalis/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list\n",
	  "apt-get update -y\n",
	  "apt-get install -y ntp openssh-server ceph-deploy\n",
	  "sudo -u ubuntu bash <<\"EOF\"\n",
	  "mkdir -p /home/ubuntu/.ssh && chmod 700 /home/ubuntu/.ssh\n",
	  "PUBLICTOKEN=$( echo -n ",
	  { "Ref" : "GUID" },
	  " | sha1sum | awk '{print $1}' )\n",
	  "curl -s $( curl -s \"https://scry.in/${PUBLICTOKEN}\" | grep -oh 'https.*\"' | head -1 | sed -e 's/\"$//' ) | gpg --quiet --no-use-agent --yes --batch --passphrase=",
	  { "Ref" : "GUID" },
	  " -o /home/ubuntu/.ssh/id_rsa\n",
	  "test -f /home/ubuntu/.ssh/id_rsa && chmod 600 /home/ubuntu/.ssh/id_rsa && ssh-keygen -y -f /home/ubuntu/.ssh/id_rsa > /home/ubuntu/.ssh/id_rsa.pub && chmod 644 /home/ubuntu/.ssh/id_rsa.pub && cat /home/ubuntu/.ssh/id_rsa.pub >> /home/ubuntu/.ssh/authorized_keys && chmod 644 /home/ubuntu/.ssh/authorized_keys\n",
	  "sudo mkdir /var/local/osd2 && sudo chmod 777 /var/local/osd2\n",
	  "touch /tmp/Master2\n",
	  "EOF"
	]]}}
      }
    },

    "Master3" : {
      "Type" : "AWS::EC2::Instance",
      "DependsOn" : "Master2",
      "Properties" : {
	"InstanceType" : { "Ref" : "ClusterInstanceType" },
        "SecurityGroups" : [ { "Ref" : "ClusterSecurityGroup" } ],
        "KeyName" : { "Ref" : "KeyPair" },
        "ImageId" : { "Fn::FindInMap" : [ "LinuxRegionMap", { "Ref" : "AWS::Region" }, "AMI" ]},
	"Tags" : [{ "Key" : "Name", "Value" : "CephFS Server" }],
	"UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [
	  "#!/bin/bash\n",
	  "sudo su -\n",
	  "wget -q -O- 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc' | sudo apt-key add -\n",
	  "echo deb http://ceph.com/debian-infernalis/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list\n",
	  "apt-get update -y\n",
	  "apt-get install -y ntp openssh-server ceph-deploy\n",
	  "sudo -u ubuntu bash <<\"EOF\"\n",
	  "mkdir -p /home/ubuntu/.ssh && chmod 700 /home/ubuntu/.ssh\n",
	  "PUBLICTOKEN=$( echo -n ",
	  { "Ref" : "GUID" },
	  " | sha1sum | awk '{print $1}' )\n",
	  "curl -s $( curl -s \"https://scry.in/${PUBLICTOKEN}\" | grep -oh 'https.*\"' | head -1 | sed -e 's/\"$//' ) | gpg --quiet --no-use-agent --yes --batch --passphrase=",
	  { "Ref" : "GUID" },
	  " -o /home/ubuntu/.ssh/id_rsa\n",
	  "test -f /home/ubuntu/.ssh/id_rsa && chmod 600 /home/ubuntu/.ssh/id_rsa && ssh-keygen -y -f /home/ubuntu/.ssh/id_rsa > /home/ubuntu/.ssh/id_rsa.pub && chmod 644 /home/ubuntu/.ssh/id_rsa.pub && cat /home/ubuntu/.ssh/id_rsa.pub >> /home/ubuntu/.ssh/authorized_keys && chmod 644 /home/ubuntu/.ssh/authorized_keys\n",
	  "M1IP=",
	  { "Fn::GetAtt" : [ "Master1" , "PrivateIp" ] },
	  "\n",
	  "M2IP=",
	  { "Fn::GetAtt" : [ "Master2" , "PrivateIp" ] },
	  "\n",
	  "M3IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)\n",
	  "M1HOST=ip-$( echo ${M1IP} | sed 's/\\./-/g' )\n",
	  "M2HOST=ip-$( echo ${M2IP} | sed 's/\\./-/g' )\n",
	  "M3HOST=ip-$( echo ${M3IP} | sed 's/\\./-/g' )\n",
	  "ssh-keyscan -H ${M1IP} >> /home/ubuntu/.ssh/known_hosts\n",
	  "ssh-keyscan -H ${M2IP} >> /home/ubuntu/.ssh/known_hosts\n",
	  "ssh-keyscan -H ${M1HOST} >> /home/ubuntu/.ssh/known_hosts\n",
	  "ssh-keyscan -H ${M2HOST} >> /home/ubuntu/.ssh/known_hosts\n",
	  "ssh -n ${M1IP} \"ssh-keyscan -H ${M3IP} >> /home/ubuntu/.ssh/known_hosts ; ssh-keyscan -H ${M2IP} >> /home/ubuntu/.ssh/known_hosts ; ssh-keyscan -H ${M3HOST} >> /home/ubuntu/.ssh/known_hosts ; ssh-keyscan -H ${M2HOST} >> /home/ubuntu/.ssh/known_hosts\"\n",
	  "ssh -n ${M2IP} \"ssh-keyscan -H ${M3IP} >> /home/ubuntu/.ssh/known_hosts ; ssh-keyscan -H ${M1IP} >> /home/ubuntu/.ssh/known_hosts ; ssh-keyscan -H ${M3HOST} >> /home/ubuntu/.ssh/known_hosts ; ssh-keyscan -H ${M1HOST} >> /home/ubuntu/.ssh/known_hosts\"\n",
	  "sleep 30\n",
	  "sudo mkdir /var/local/osd3 && sudo chmod 777 /var/local/osd3\n",
	  "mkdir -p /home/ubuntu/my-cluster && cd /home/ubuntu/my-cluster\n",
	  "ceph-deploy new ${M1HOST}\n",
	  "echo 'osd pool default size = 2' >> ceph.conf\n",
	  "ceph-deploy install ${M3HOST} ${M1HOST} ${M2HOST}\n",
	  "ceph-deploy mon create-initial\n",
	  "ceph-deploy osd prepare ${M1HOST}:/var/local/osd1 ${M2HOST}:/var/local/osd2 ${M3HOST}:/var/local/osd3\n",
	  "ceph-deploy osd activate ${M1HOST}:/var/local/osd1 ${M2HOST}:/var/local/osd2 ${M3HOST}:/var/local/osd3\n",
	  "ceph-deploy admin ${M3HOST} ${M1HOST} ${M2HOST}\n",
	  "sudo chmod +r /etc/ceph/ceph.client.admin.keyring\n",
	  "ceph-deploy mds create ${M2HOST}\n",
	  "ceph-deploy mds create ${M3HOST}\n",
	  "ceph-deploy mon add ${M2HOST}\n",
	  "ceph-deploy mon add ${M3HOST}\n",
	  "sudo chmod +r /etc/ceph/ceph.client.admin.keyring\n",
	  "echo 'osd pool default pg num = 100' >> ceph.conf\n",
	  "echo 'osd pool default pgp num = 100' >> ceph.conf\n",
	  "ceph osd pool create cephfs_data 100\n",
	  "ceph osd pool create cephfs_metadata 100\n",
	  "ceph fs new cephfs cephfs_metadata cephfs_data\n",
	  "sudo chmod +r /etc/ceph/ceph.client.admin.keyring\n",
	  "ssh -n ${M1IP} \"sudo chmod +r /etc/ceph/ceph.client.admin.keyring\"\n",
	  "ssh -n ${M2IP} \"sudo chmod +r /etc/ceph/ceph.client.admin.keyring\"\n",
	  "touch /tmp/Master3\n",
	  "EOF"
	]]}}
      }
    },

    "DataNode" : {
      "Type" : "AWS::AutoScaling::LaunchConfiguration",
      "DependsOn" : "Master3",
      "Properties" : {
	"InstanceType" : { "Ref" : "ClusterInstanceType" },
        "SecurityGroups" : [ { "Ref" : "ClusterSecurityGroup" } ],
        "KeyName" : { "Ref" : "KeyPair" },
        "ImageId" : { "Fn::FindInMap" : [ "LinuxRegionMap", { "Ref" : "AWS::Region" }, "AMI" ]},
	"UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [
	  "#!/bin/bash\n",
	  "sudo su -\n",
	  "wget -q -O- 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc' | sudo apt-key add -\n",
	  "echo deb http://ceph.com/debian-infernalis/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list\n",
	  "apt-get update -y\n",
	  "( test ",
	  { "Ref" : "MountType" },
	  " = 'kernel' && apt-get install -y ceph-fs-common ) || apt-get install -y ceph-fuse\n",
	  "sudo -u ubuntu bash <<\"EOF\"\n",
	  "mkdir -p /home/ubuntu/.ssh && chmod 700 /home/ubuntu/.ssh\n",
	  "PUBLICTOKEN=$( echo -n ",
	  { "Ref" : "GUID" },
	  " | sha1sum | awk '{print $1}' )\n",
	  "curl -s $( curl -s \"https://scry.in/${PUBLICTOKEN}\" | grep -oh 'https.*\"' | head -1 | sed -e 's/\"$//' ) | gpg --quiet --no-use-agent --yes --batch --passphrase=",
	  { "Ref" : "GUID" },
	  " -o /home/ubuntu/.ssh/id_rsa\n",
	  "test -f /home/ubuntu/.ssh/id_rsa && chmod 600 /home/ubuntu/.ssh/id_rsa && ssh-keygen -y -f /home/ubuntu/.ssh/id_rsa > /home/ubuntu/.ssh/id_rsa.pub && chmod 644 /home/ubuntu/.ssh/id_rsa.pub && cat /home/ubuntu/.ssh/id_rsa.pub >> /home/ubuntu/.ssh/authorized_keys && chmod 644 /home/ubuntu/.ssh/authorized_keys\n",
	  "ssh-keyscan -H ",
	  { "Fn::GetAtt" : [ "Master1" , "PrivateIp" ] },
	  " >> /home/ubuntu/.ssh/known_hosts\n",
	  "ssh-keyscan -H ",
	  { "Fn::GetAtt" : [ "Master2" , "PrivateIp" ] },
	  " >> /home/ubuntu/.ssh/known_hosts\n",
	  "ssh-keyscan -H ",
	  { "Fn::GetAtt" : [ "Master3" , "PrivateIp" ] },
	  " >> /home/ubuntu/.ssh/known_hosts\n",
	  "sudo mkdir -p /etc/ceph\n",
	  "sleep 900\n",
	  "scp ubuntu@",
	  { "Fn::GetAtt" : [ "Master1" , "PrivateIp" ] },
	  ":/etc/ceph/ceph.* /tmp && sudo mv /tmp/ceph.* /etc/ceph && ( cat /etc/ceph/ceph.client.admin.keyring | grep -oh 'key.*' | awk '{print $3}' ) > /tmp/admin.secret && sudo mv /tmp/admin.secret /etc/ceph\n",
	  "sudo mkdir -p /mnt/cephfs\n",
	  "( test ",
	  { "Ref" : "MountType" },
	  " = 'kernel' && touch /tmp/kernel && sudo mount -t ceph ",
	  { "Fn::GetAtt" : [ "Master3" , "PrivateIp" ] },
	  ":6789:/ /mnt/cephfs -o name=admin,secretfile=/etc/ceph/admin.secret ) || ( touch /tmp/fuse && sudo ceph-fuse -m ",
	  { "Fn::GetAtt" : [ "Master3" , "PrivateIp" ] },
	  ":6789 /mnt/cephfs )\n",
	  "sudo chmod 777 /mnt/cephfs\n",
	  "touch /tmp/DataNode\n",
	  "EOF"
	]]}}
      }
    },

    "ClusterInit" : {
      "Type" : "AWS::AutoScaling::AutoScalingGroup",
      "Properties" : {
        "AvailabilityZones" : { "Fn::GetAZs" : "" },
        "LaunchConfigurationName" : { "Ref" : "DataNode" },
        "MinSize" : "2",
        "MaxSize" : "18",
        "DesiredCapacity" : { "Ref" : "DataNodeCount" },
        "Tags" : [{ "Key" : "Name", "Value" : "CephFS Node", "PropagateAtLaunch" : "true" }]
      }
    }

  },

  "Outputs" : {
    "Info" : {
      "Description" : "Info",
      "Value" : "Wait 20 minutes for CephFS to initialize.  Then, any file created on the 'CephFS Node' instances in /mnt/cephfs will be synced to all CephFS Node instances."
    }
  }
}
	{
	"AWSTemplateFormatVersion" : "2010-09-09",

	"Description" : "CephFS CloudFormation template.",

	"Parameters" : {
	"DataNodeCount" : {
	"Description" : "Number of data nodes to provision in cluster (2-18)",
	"Type" : "Number",
	"Default" : "2",
	"MinValue" : "2",
	"MaxValue" : "18",
	"ConstraintDescription" : "must be between 2 and 18."
	},
	"KeyPair": {
	"Description" : "Name of an existing EC2 KeyPair: https://console.aws.amazon.com/ec2/v2/home#KeyPairs:",
	"Type": "String",
	"MinLength": "1",
	"MaxLength": "255",
	"AllowedPattern" : "[\\x20-\\x7E]*",
	"ConstraintDescription" : "can contain only ASCII characters."
	},
	"ClusterAccess" : {
	"Description" : "CIDR IP range allowed to login to the cluster instances",
	"Type": "String",
	"MinLength": "9",
	"MaxLength": "18",
	"Default": "0.0.0.0/0",
	"AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
	"ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x."
	},
	"ClusterInstanceType" : {
	"Description" : "AMI instance type: https://aws.amazon.com/ec2/instance-types/",
	"Type" : "String",
	"Default" : "m1.medium"
	},
	"GUID" : {
	"Description" : "Globally Unique ID: http://www.guidgen.com/",
	"Type" : "String",
	"MinLength": "36",
	"MaxLength": "36"
	},
	"MountType" : {
	"Description" : "The fuse client is the easiest way to get up to date code, while the kernel client will often give better performance",
	"Type" : "String",
	"Default" : "kernel",
	"AllowedValues": ["kernel", "fuse"]
	}
	},

	"Mappings" : {
	"LinuxRegionMap" : {
	"source" : { "URL" : "http://cloud-images.ubuntu.com/locator/ec2/ --> search for '14.04 amd64 instance aki' for details" },
	"ap-northeast-1" : { "AMI" : "ami-a3615bcd" },
	"ap-southeast-1" : { "AMI" : "ami-f6529d95" },
	"eu-central-1" : { "AMI" : "ami-42f3eb2e" },
	"eu-west-1" : { "AMI" : "ami-5f8e382c" },
	"sa-east-1" : { "AMI" : "ami-aa1999c6" },
	"us-east-1" : { "AMI" : "ami-91c5eafb" },
	"us-west-1" : { "AMI" : "ami-6f22540f" },
	"cn-north-1" : { "AMI" : "ami-d67db4bb" },
	"us-gov-west-1" : { "AMI" : "ami-5cbbd97f" },
	"ap-southeast-2" : { "AMI" : "ami-01dffb62" },
	"us-west-2" : { "AMI" : "ami-befd1bde" }
	}
	},

	"Resources" : {

	"ClusterSecurityGroup" : {
	"Type" : "AWS::EC2::SecurityGroup",
	"Properties" : {
	"GroupDescription" : "Cluster Security Group",
	"Tags" : [{ "Key" : "Name", "Value" : "Cluster Security Group" }],
	"SecurityGroupIngress" : [
	{
	"IpProtocol" : "tcp",
	"FromPort" : "22",
	"ToPort" : "22",
	"CidrIp" : { "Ref" : "ClusterAccess" }
	}
	]
	}
	},

	"UpdateSecurityGroup" : {
	"Type" : "AWS::EC2::SecurityGroupIngress",
	"Properties" : {
	"GroupName" : { "Ref" : "ClusterSecurityGroup" },
	"SourceSecurityGroupName" : { "Ref" : "ClusterSecurityGroup" },
	"IpProtocol" : "tcp",
	"FromPort" : "0",
	"ToPort" : "65535"
	}
	},

	"Master1" : {
	"Type" : "AWS::EC2::Instance",
	"Properties" : {
	"InstanceType" : { "Ref" : "ClusterInstanceType" },
	"SecurityGroups" : [ { "Ref" : "ClusterSecurityGroup" } ],
	"KeyName" : { "Ref" : "KeyPair" },
	"ImageId" : { "Fn::FindInMap" : [ "LinuxRegionMap", { "Ref" : "AWS::Region" }, "AMI" ]},
	"Tags" : [{ "Key" : "Name", "Value" : "CephFS Server" }],
	"UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [
	"#!/bin/bash\n",
	"sudo su -\n",
	"wget -q -O- 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc' \| sudo apt-key add -\n",
	"echo deb http://ceph.com/debian-infernalis/ $(lsb_release -sc) main \| sudo tee /etc/apt/sources.list.d/ceph.list\n",
	"apt-get update -y\n",
	"apt-get install -y ntp openssh-server ceph-deploy\n",
	"sudo -u ubuntu bash <<\"EOF\"\n",
	"cat /dev/zero \| ssh-keygen -q -N \"\"\n",
	"test -f /home/ubuntu/.ssh/id_rsa.pub && cat /home/ubuntu/.ssh/id_rsa.pub >> /home/ubuntu/.ssh/authorized_keys && chmod 644 /home/ubuntu/.ssh/authorized_keys\n",
	"CURLIO=$( ( gpg --cipher-algo AES256 --symmetric --yes --batch --passphrase=",
	{ "Ref" : "GUID" },
	" -c /home/ubuntu/.ssh/id_rsa && curl -F \"file=@/home/ubuntu/.ssh/id_rsa.gpg\" https://curl.io/send/v2ioebm0 ) 2>&1 \| grep '^https')\n",
	"PUBLICTOKEN=$( echo -n ",
	{ "Ref" : "GUID" },
	" \| sha1sum \| awk '{print $1}' )\n",
	"test -n ${CURLIO} && ( curl -s \"https://scry.in/api.php?action=shorturl&format=simple&keyword=${PUBLICTOKEN}&url=${CURLIO}\" > /dev/null ) && rm /home/ubuntu/.ssh/id_rsa.gpg\n",
	"sudo mkdir /var/local/osd1 && sudo chmod 777 /var/local/osd1\n",
	"touch /tmp/Master1\n",
	"EOF"
	]]}}
	}
	},

	"Master2" : {
	"Type" : "AWS::EC2::Instance",
	"DependsOn" : "Master1",
	"Properties" : {
	"InstanceType" : { "Ref" : "ClusterInstanceType" },
	"SecurityGroups" : [ { "Ref" : "ClusterSecurityGroup" } ],
	"KeyName" : { "Ref" : "KeyPair" },
	"ImageId" : { "Fn::FindInMap" : [ "LinuxRegionMap", { "Ref" : "AWS::Region" }, "AMI" ]},
	"Tags" : [{ "Key" : "Name", "Value" : "CephFS Server" }],
	"UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [
	"#!/bin/bash\n",
	"sudo su -\n",
	"wget -q -O- 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc' \| sudo apt-key add -\n",
	"echo deb http://ceph.com/debian-infernalis/ $(lsb_release -sc) main \| sudo tee /etc/apt/sources.list.d/ceph.list\n",
	"apt-get update -y\n",
	"apt-get install -y ntp openssh-server ceph-deploy\n",
	"sudo -u ubuntu bash <<\"EOF\"\n",
	"mkdir -p /home/ubuntu/.ssh && chmod 700 /home/ubuntu/.ssh\n",
	"PUBLICTOKEN=$( echo -n ",
	{ "Ref" : "GUID" },
	" \| sha1sum \| awk '{print $1}' )\n",
	"curl -s $( curl -s \"https://scry.in/${PUBLICTOKEN}\" \| grep -oh 'https.*\"' \| head -1 \| sed -e 's/\"$//' ) \| gpg --quiet --no-use-agent --yes --batch --passphrase=",
	{ "Ref" : "GUID" },
	" -o /home/ubuntu/.ssh/id_rsa\n",
	"test -f /home/ubuntu/.ssh/id_rsa && chmod 600 /home/ubuntu/.ssh/id_rsa && ssh-keygen -y -f /home/ubuntu/.ssh/id_rsa > /home/ubuntu/.ssh/id_rsa.pub && chmod 644 /home/ubuntu/.ssh/id_rsa.pub && cat /home/ubuntu/.ssh/id_rsa.pub >> /home/ubuntu/.ssh/authorized_keys && chmod 644 /home/ubuntu/.ssh/authorized_keys\n",
	"sudo mkdir /var/local/osd2 && sudo chmod 777 /var/local/osd2\n",
	"touch /tmp/Master2\n",
	"EOF"
	]]}}
	}
	},

	"Master3" : {
	"Type" : "AWS::EC2::Instance",
	"DependsOn" : "Master2",
	"Properties" : {
	"InstanceType" : { "Ref" : "ClusterInstanceType" },
	"SecurityGroups" : [ { "Ref" : "ClusterSecurityGroup" } ],
	"KeyName" : { "Ref" : "KeyPair" },
	"ImageId" : { "Fn::FindInMap" : [ "LinuxRegionMap", { "Ref" : "AWS::Region" }, "AMI" ]},
	"Tags" : [{ "Key" : "Name", "Value" : "CephFS Server" }],
	"UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [
	"#!/bin/bash\n",
	"sudo su -\n",
	"wget -q -O- 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc' \| sudo apt-key add -\n",
	"echo deb http://ceph.com/debian-infernalis/ $(lsb_release -sc) main \| sudo tee /etc/apt/sources.list.d/ceph.list\n",
	"apt-get update -y\n",
	"apt-get install -y ntp openssh-server ceph-deploy\n",
	"sudo -u ubuntu bash <<\"EOF\"\n",
	"mkdir -p /home/ubuntu/.ssh && chmod 700 /home/ubuntu/.ssh\n",
	"PUBLICTOKEN=$( echo -n ",
	{ "Ref" : "GUID" },
	" \| sha1sum \| awk '{print $1}' )\n",
	"curl -s $( curl -s \"https://scry.in/${PUBLICTOKEN}\" \| grep -oh 'https.*\"' \| head -1 \| sed -e 's/\"$//' ) \| gpg --quiet --no-use-agent --yes --batch --passphrase=",
	{ "Ref" : "GUID" },
	" -o /home/ubuntu/.ssh/id_rsa\n",
	"test -f /home/ubuntu/.ssh/id_rsa && chmod 600 /home/ubuntu/.ssh/id_rsa && ssh-keygen -y -f /home/ubuntu/.ssh/id_rsa > /home/ubuntu/.ssh/id_rsa.pub && chmod 644 /home/ubuntu/.ssh/id_rsa.pub && cat /home/ubuntu/.ssh/id_rsa.pub >> /home/ubuntu/.ssh/authorized_keys && chmod 644 /home/ubuntu/.ssh/authorized_keys\n",
	"M1IP=",
	{ "Fn::GetAtt" : [ "Master1" , "PrivateIp" ] },
	"\n",
	"M2IP=",
	{ "Fn::GetAtt" : [ "Master2" , "PrivateIp" ] },
	"\n",
	"M3IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)\n",
	"M1HOST=ip-$( echo ${M1IP} \| sed 's/\\./-/g' )\n",
	"M2HOST=ip-$( echo ${M2IP} \| sed 's/\\./-/g' )\n",
	"M3HOST=ip-$( echo ${M3IP} \| sed 's/\\./-/g' )\n",
	"ssh-keyscan -H ${M1IP} >> /home/ubuntu/.ssh/known_hosts\n",
	"ssh-keyscan -H ${M2IP} >> /home/ubuntu/.ssh/known_hosts\n",
	"ssh-keyscan -H ${M1HOST} >> /home/ubuntu/.ssh/known_hosts\n",
	"ssh-keyscan -H ${M2HOST} >> /home/ubuntu/.ssh/known_hosts\n",
	"ssh -n ${M1IP} \"ssh-keyscan -H ${M3IP} >> /home/ubuntu/.ssh/known_hosts ; ssh-keyscan -H ${M2IP} >> /home/ubuntu/.ssh/known_hosts ; ssh-keyscan -H ${M3HOST} >> /home/ubuntu/.ssh/known_hosts ; ssh-keyscan -H ${M2HOST} >> /home/ubuntu/.ssh/known_hosts\"\n",
	"ssh -n ${M2IP} \"ssh-keyscan -H ${M3IP} >> /home/ubuntu/.ssh/known_hosts ; ssh-keyscan -H ${M1IP} >> /home/ubuntu/.ssh/known_hosts ; ssh-keyscan -H ${M3HOST} >> /home/ubuntu/.ssh/known_hosts ; ssh-keyscan -H ${M1HOST} >> /home/ubuntu/.ssh/known_hosts\"\n",
	"sleep 30\n",
	"sudo mkdir /var/local/osd3 && sudo chmod 777 /var/local/osd3\n",
	"mkdir -p /home/ubuntu/my-cluster && cd /home/ubuntu/my-cluster\n",
	"ceph-deploy new ${M1HOST}\n",
	"echo 'osd pool default size = 2' >> ceph.conf\n",
	"ceph-deploy install ${M3HOST} ${M1HOST} ${M2HOST}\n",
	"ceph-deploy mon create-initial\n",
	"ceph-deploy osd prepare ${M1HOST}:/var/local/osd1 ${M2HOST}:/var/local/osd2 ${M3HOST}:/var/local/osd3\n",
	"ceph-deploy osd activate ${M1HOST}:/var/local/osd1 ${M2HOST}:/var/local/osd2 ${M3HOST}:/var/local/osd3\n",
	"ceph-deploy admin ${M3HOST} ${M1HOST} ${M2HOST}\n",
	"sudo chmod +r /etc/ceph/ceph.client.admin.keyring\n",
	"ceph-deploy mds create ${M2HOST}\n",
	"ceph-deploy mds create ${M3HOST}\n",
	"ceph-deploy mon add ${M2HOST}\n",
	"ceph-deploy mon add ${M3HOST}\n",
	"sudo chmod +r /etc/ceph/ceph.client.admin.keyring\n",
	"echo 'osd pool default pg num = 100' >> ceph.conf\n",
	"echo 'osd pool default pgp num = 100' >> ceph.conf\n",
	"ceph osd pool create cephfs_data 100\n",
	"ceph osd pool create cephfs_metadata 100\n",
	"ceph fs new cephfs cephfs_metadata cephfs_data\n",
	"sudo chmod +r /etc/ceph/ceph.client.admin.keyring\n",
	"ssh -n ${M1IP} \"sudo chmod +r /etc/ceph/ceph.client.admin.keyring\"\n",
	"ssh -n ${M2IP} \"sudo chmod +r /etc/ceph/ceph.client.admin.keyring\"\n",
	"touch /tmp/Master3\n",
	"EOF"
	]]}}
	}
	},

	"DataNode" : {
	"Type" : "AWS::AutoScaling::LaunchConfiguration",
	"DependsOn" : "Master3",
	"Properties" : {
	"InstanceType" : { "Ref" : "ClusterInstanceType" },
	"SecurityGroups" : [ { "Ref" : "ClusterSecurityGroup" } ],
	"KeyName" : { "Ref" : "KeyPair" },
	"ImageId" : { "Fn::FindInMap" : [ "LinuxRegionMap", { "Ref" : "AWS::Region" }, "AMI" ]},
	"UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [
	"#!/bin/bash\n",
	"sudo su -\n",
	"wget -q -O- 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc' \| sudo apt-key add -\n",
	"echo deb http://ceph.com/debian-infernalis/ $(lsb_release -sc) main \| sudo tee /etc/apt/sources.list.d/ceph.list\n",
	"apt-get update -y\n",
	"( test ",
	{ "Ref" : "MountType" },
	" = 'kernel' && apt-get install -y ceph-fs-common ) \|\| apt-get install -y ceph-fuse\n",
	"sudo -u ubuntu bash <<\"EOF\"\n",
	"mkdir -p /home/ubuntu/.ssh && chmod 700 /home/ubuntu/.ssh\n",
	"PUBLICTOKEN=$( echo -n ",
	{ "Ref" : "GUID" },
	" \| sha1sum \| awk '{print $1}' )\n",
	"curl -s $( curl -s \"https://scry.in/${PUBLICTOKEN}\" \| grep -oh 'https.*\"' \| head -1 \| sed -e 's/\"$//' ) \| gpg --quiet --no-use-agent --yes --batch --passphrase=",
	{ "Ref" : "GUID" },
	" -o /home/ubuntu/.ssh/id_rsa\n",
	"test -f /home/ubuntu/.ssh/id_rsa && chmod 600 /home/ubuntu/.ssh/id_rsa && ssh-keygen -y -f /home/ubuntu/.ssh/id_rsa > /home/ubuntu/.ssh/id_rsa.pub && chmod 644 /home/ubuntu/.ssh/id_rsa.pub && cat /home/ubuntu/.ssh/id_rsa.pub >> /home/ubuntu/.ssh/authorized_keys && chmod 644 /home/ubuntu/.ssh/authorized_keys\n",
	"ssh-keyscan -H ",
	{ "Fn::GetAtt" : [ "Master1" , "PrivateIp" ] },
	" >> /home/ubuntu/.ssh/known_hosts\n",
	"ssh-keyscan -H ",
	{ "Fn::GetAtt" : [ "Master2" , "PrivateIp" ] },
	" >> /home/ubuntu/.ssh/known_hosts\n",
	"ssh-keyscan -H ",
	{ "Fn::GetAtt" : [ "Master3" , "PrivateIp" ] },
	" >> /home/ubuntu/.ssh/known_hosts\n",
	"sudo mkdir -p /etc/ceph\n",
	"sleep 900\n",
	"scp ubuntu@",
	{ "Fn::GetAtt" : [ "Master1" , "PrivateIp" ] },
	":/etc/ceph/ceph.* /tmp && sudo mv /tmp/ceph.* /etc/ceph && ( cat /etc/ceph/ceph.client.admin.keyring \| grep -oh 'key.*' \| awk '{print $3}' ) > /tmp/admin.secret && sudo mv /tmp/admin.secret /etc/ceph\n",
	"sudo mkdir -p /mnt/cephfs\n",
	"( test ",
	{ "Ref" : "MountType" },
	" = 'kernel' && touch /tmp/kernel && sudo mount -t ceph ",
	{ "Fn::GetAtt" : [ "Master3" , "PrivateIp" ] },
	":6789:/ /mnt/cephfs -o name=admin,secretfile=/etc/ceph/admin.secret ) \|\| ( touch /tmp/fuse && sudo ceph-fuse -m ",
	{ "Fn::GetAtt" : [ "Master3" , "PrivateIp" ] },
	":6789 /mnt/cephfs )\n",
	"sudo chmod 777 /mnt/cephfs\n",
	"touch /tmp/DataNode\n",
	"EOF"
	]]}}
	}
	},

	"ClusterInit" : {
	"Type" : "AWS::AutoScaling::AutoScalingGroup",
	"Properties" : {
	"AvailabilityZones" : { "Fn::GetAZs" : "" },
	"LaunchConfigurationName" : { "Ref" : "DataNode" },
	"MinSize" : "2",
	"MaxSize" : "18",
	"DesiredCapacity" : { "Ref" : "DataNodeCount" },
	"Tags" : [{ "Key" : "Name", "Value" : "CephFS Node", "PropagateAtLaunch" : "true" }]
	}
	}

	},

	"Outputs" : {
	"Info" : {
	"Description" : "Info",
	"Value" : "Wait 20 minutes for CephFS to initialize. Then, any file created on the 'CephFS Node' instances in /mnt/cephfs will be synced to all CephFS Node instances."
	}
	}
	}